tests/test_edgar.py

import pytest
from unittest.mock import patch, MagicMock
from ingestion.edgar import (
    get_cik,
    compute_metrics_for_accn,
    get_all_xbrl_facts,
    _extract_business,
    _extract_guidance,
    _extract_mda,
    _extract_risk_factors,
    _extract_segments_geography,
    _html_to_filing_text,
)

FAKE_TICKERS = {
    "0": {"cik_str": 320193, "ticker": "AAPL", "title": "Apple Inc."}
}

ACCN = "0000320193-24-000123"
ACCN_PRIOR = "0000320193-23-000456"

FAKE_XBRL = {
    "cik": 320193, "entityName": "Apple Inc.",
    "facts": {"us-gaap": {
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            {"accn": ACCN_PRIOR, "end": "2023-09-30", "val": 383285000000, "form": "10-K", "fp": "FY", "fy": 2023, "filed": "2023-11-03"},
            {"accn": ACCN, "end": "2024-09-28", "val": 391035000000, "form": "10-K", "fp": "FY", "fy": 2024, "filed": "2024-11-01"},
        ]}},
        "GrossProfit": {"units": {"USD": [
            {"accn": ACCN, "end": "2024-09-28", "val": 180683000000, "form": "10-K", "fp": "FY", "fy": 2024, "filed": "2024-11-01"},
        ]}},
        "OperatingIncomeLoss": {"units": {"USD": [
            {"accn": ACCN, "end": "2024-09-28", "val": 123216000000, "form": "10-K", "fp": "FY", "fy": 2024, "filed": "2024-11-01"},
        ]}},
        "EarningsPerShareDiluted": {"units": {"USD/shares": [
            {"accn": ACCN, "end": "2024-09-28", "val": 6.11, "form": "10-K", "fp": "FY", "fy": 2024, "filed": "2024-11-01"},
        ]}},
        "NetCashProvidedByUsedInOperatingActivities": {"units": {"USD": [
            {"accn": ACCN, "end": "2024-09-28", "val": 118254000000, "form": "10-K", "fp": "FY", "fy": 2024, "filed": "2024-11-01"},
        ]}},
        "PaymentsToAcquirePropertyPlantAndEquipment": {"units": {"USD": [
            {"accn": ACCN, "end": "2024-09-28", "val": 9447000000, "form": "10-K", "fp": "FY", "fy": 2024, "filed": "2024-11-01"},
        ]}},
    }}
}


def _mock_resp(json_data):
    m = MagicMock()
    m.json.return_value = json_data
    m.raise_for_status = MagicMock()
    return m


@patch("ingestion.edgar.httpx.get")
def test_get_cik_known_ticker(mock_get):
    mock_get.return_value = _mock_resp(FAKE_TICKERS)
    assert get_cik("AAPL") == "0000320193"


@patch("ingestion.edgar.httpx.get")
def test_get_cik_unknown_ticker(mock_get):
    mock_get.return_value = _mock_resp(FAKE_TICKERS)
    assert get_cik("ZZZZ") is None


def test_compute_metrics_revenue():
    facts = FAKE_XBRL["facts"]["us-gaap"]
    m = compute_metrics_for_accn(facts, ACCN, "10-K")
    assert m["revenue"] == 391035000000.0
    assert abs(m["revenue_yoy_pct"] - 2.02) < 0.1


def test_compute_metrics_margins():
    facts = FAKE_XBRL["facts"]["us-gaap"]
    m = compute_metrics_for_accn(facts, ACCN, "10-K")
    assert m["gross_margin"] == pytest.approx(180683000000 / 391035000000, rel=1e-3)
    assert m["eps"] == 6.11
    assert m["free_cash_flow"] == 118254000000 - 9447000000


def test_compute_metrics_period_string():
    facts = FAKE_XBRL["facts"]["us-gaap"]
    m = compute_metrics_for_accn(facts, ACCN, "10-K")
    assert m["period"] == "FY2024"


def test_compute_metrics_unknown_accn_returns_none_values():
    facts = FAKE_XBRL["facts"]["us-gaap"]
    m = compute_metrics_for_accn(facts, "0000000000-00-000000", "10-K")
    assert m["revenue"] is None
    assert m["period"] == ""


def test_extract_guidance_finds_sentences():
    mda = (
        "The company expects revenue in the range of $10 billion to $11 billion for the next quarter. "
        "We anticipate gross margin of approximately 43%, driven by product mix. "
        "Competition remains fierce. Cost pressures are ongoing."
    )
    result = _extract_guidance(mda)
    assert result is not None
    assert "expects" in result.lower() or "anticipate" in result.lower()


def test_extract_guidance_returns_none_when_no_match():
    mda = "Revenue was strong. Costs were controlled. Competition increased."
    assert _extract_guidance(mda) is None


def test_extract_guidance_empty_input():
    assert _extract_guidance("") is None


# ── _extract_mda / _extract_risk_factors ────────────────────────────────────

MDA_10K_TEXT = """\
PART II

Item 6. Selected Financial Data

...financial tables...

Item 7. Management’s Discussion and Analysis of Financial Condition and Results of Operations

Revenue increased 5% year-over-year driven by product mix improvements.
Operating margin expanded 200 basis points to 32%.
Free cash flow was $110 billion.

Item 7A. Quantitative and Qualitative Disclosures About Market Risk

Interest rate risk exposure is managed through derivatives.
"""

MDA_10K_TOC_TEXT = """\
Table of Contents

Item 7. Management's Discussion and Analysis.......34
Item 7A. Quantitative Disclosures.......................67

Item 7. Management’s Discussion and Analysis of Financial Condition

Revenue grew 8% year-over-year to $400 billion.
Gross margin was 46%, up from 44%.

Item 7A. Quantitative and Qualitative Disclosures

Interest rate risk content here.
"""

MDA_10Q_TEXT = """\
PART I - FINANCIAL INFORMATION

Item 1. Financial Statements

...tables...

Item 2. Management’s Discussion and Analysis of Financial Condition and Results of Operations

Revenue was $30.0 billion, up 5% year-over-year.
Services segment grew 12% to $25 billion.
Gross margin was 43.8%.

Item 3. Quantitative and Qualitative Disclosures About Market Risk

Market risk content here.
"""

RISK_TEXT = """\
Item 1. Business

Apple designs and manufactures consumer electronics.

Item 1A. Risk Factors

Macroeconomic conditions may affect consumer spending.
Competition in the smartphone market remains intense.
Supply chain disruptions could impact production.

Item 1B. Unresolved Staff Comments

None.

Item 2. Properties
"""


def test_extract_mda_10k_finds_item7_content():
    result = _extract_mda(MDA_10K_TEXT, "10-K")
    assert "Revenue increased 5%" in result
    assert "Operating margin expanded" in result


def test_extract_mda_10k_excludes_item7a_content():
    result = _extract_mda(MDA_10K_TEXT, "10-K")
    assert "Interest rate risk exposure" not in result


def test_extract_mda_10k_uses_last_occurrence_not_toc():
    result = _extract_mda(MDA_10K_TOC_TEXT, "10-K")
    # content after the SECOND "Item 7" should be captured, not just the TOC page ref
    assert "Revenue grew 8%" in result
    assert "Gross margin was 46%" in result


def test_extract_mda_10q_finds_item2_content():
    result = _extract_mda(MDA_10Q_TEXT, "10-Q")
    assert "Revenue was $30.0 billion" in result
    assert "Services segment grew 12%" in result


def test_extract_mda_10q_excludes_item3_content():
    result = _extract_mda(MDA_10Q_TEXT, "10-Q")
    assert "Market risk content" not in result


def test_extract_mda_returns_empty_when_not_found():
    result = _extract_mda("Some text with no relevant headings.", "10-K")
    assert result == ""


def test_extract_mda_10q_returns_empty_when_not_found():
    result = _extract_mda("Some text with no relevant headings.", "10-Q")
    assert result == ""


def test_extract_mda_respects_80k_cap():
    long_text = (
        "Item 7. Management’s Discussion and Analysis\n\n"
        + "Revenue content. " * 10000
        + "\nItem 7A. Quantitative\n"
    )
    result = _extract_mda(long_text, "10-K")
    assert len(result) <= 80_000


def test_extract_risk_factors_finds_item1a_content():
    result = _extract_risk_factors(RISK_TEXT)
    assert "Macroeconomic conditions" in result
    assert "Competition in the smartphone market" in result


def test_extract_risk_factors_excludes_item1b_content():
    result = _extract_risk_factors(RISK_TEXT)
    assert "Unresolved Staff Comments" not in result


def test_extract_risk_factors_returns_empty_when_not_found():
    result = _extract_risk_factors("Some text without any risk factors heading.")
    assert result == ""


def test_extract_business_uses_item_1_and_stops_before_risks():
    text = """Table of Contents
Item 1. Business
Item 1A. Risk Factors

Item 1. Business
We design accelerated computing platforms for data centers.
Our customers include cloud service providers.

Item 1A. Risk Factors
Export controls may affect sales.
"""
    result = _extract_business(text, "10-K")
    assert "accelerated computing platforms" in result
    assert "Export controls" not in result


def test_extract_business_returns_empty_for_10q():
    assert _extract_business("Item 1. Business\nText\nItem 1A.", "10-Q") == ""


def test_extract_segments_geography_preserves_disclosed_table_text():
    text = """Item 8. Financial Statements and Supplementary Data

Note 12 - Segment Information
Compute | 80 | 70
Networking | 20 | 30

Geographic Information
United States | 60%
International | 40%

Item 9. Changes in and Disagreements with Accountants
"""
    result = _extract_segments_geography(text, "10-K")
    assert "Compute | 80 | 70" in result
    assert "United States | 60%" in result
    assert "Changes in and Disagreements" not in result


def test_extract_segments_geography_handles_combined_heading_after_item_9():
    text = """Item 8. Financial Statements and Supplementary Data
See Item 15 for the audited notes.
Item 9. Changes in and Disagreements with Accountants
Item 15. Exhibits and Financial Statement Schedules

Note 18 - Segment Information and Geographic Data
Revenue, classified by the major geographic areas, was as follows:
United States | 120
Other countries | 80
"""
    result = _extract_segments_geography(text, "10-K")
    assert "Segment Information and Geographic Data" in result
    assert "United States | 120" in result


def test_html_to_filing_text_keeps_table_rows():
    html = """
    <html><body><h2>Geographic Information</h2>
    <table><tr><th>Region</th><th>Revenue</th></tr>
    <tr><td>Europe</td><td>25%</td></tr></table></body></html>
    """
    result = _html_to_filing_text(html)
    assert "Region | Revenue" in result
    assert "Europe | 25%" in result


# ── XBRL concept expansion ───────────────────────────────────────────────────

def test_compute_metrics_uses_sales_revenue_net_concept():
    """Legacy SalesRevenueNet concept (ASC 605, pre-2018) is picked up."""
    facts = {
        "SalesRevenueNet": {"units": {"USD": [
            {"accn": ACCN, "end": "2018-12-31", "val": 50_000_000_000,
             "form": "10-K", "fp": "FY", "fy": 2018, "filed": "2019-02-15"},
        ]}}
    }
    m = compute_metrics_for_accn(facts, ACCN, "10-K")
    assert m["revenue"] == 50_000_000_000.0


def test_compute_metrics_uses_revenue_net_concept():
    """Fallback concept RevenueNet is tried when primary concepts are absent."""
    facts = {
        "RevenueNet": {"units": {"USD": [
            {"accn": ACCN, "end": "2020-12-31", "val": 60_000_000_000,
             "form": "10-K", "fp": "FY", "fy": 2020, "filed": "2021-02-15"},
        ]}}
    }
    m = compute_metrics_for_accn(facts, ACCN, "10-K")
    assert m["revenue"] == 60_000_000_000.0


# ── EPS short-circuit fix ────────────────────────────────────────────────────

def test_compute_metrics_eps_falls_back_to_basic_when_diluted_accn_missing():
    """EPS uses EarningsPerShareBasic when Diluted exists but has no entry for this accn."""
    OTHER_ACCN = "0000000000-99-000000"
    facts = {
        "EarningsPerShareDiluted": {"units": {"USD/shares": [
            {"accn": OTHER_ACCN, "end": "2024-09-28", "val": 6.11,
             "form": "10-K", "fp": "FY", "fy": 2024, "filed": "2024-11-01"},
        ]}},
        "EarningsPerShareBasic": {"units": {"USD/shares": [
            {"accn": ACCN, "end": "2024-09-28", "val": 6.15,
             "form": "10-K", "fp": "FY", "fy": 2024, "filed": "2024-11-01"},
        ]}},
    }
    m = compute_metrics_for_accn(facts, ACCN, "10-K")
    assert m["eps"] == 6.15


# ── Report-date fallback ─────────────────────────────────────────────────────

def test_compute_metrics_report_date_fallback_fills_revenue():
    """When accn has no XBRL match, report_date within ±15 days of entry 'end' is used."""
    facts = {
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            {"accn": "0000000-other-accn", "end": "2025-09-28", "val": 35_000_000_000,
             "form": "10-Q", "fp": "Q3", "fy": 2025, "filed": "2025-10-30"},
        ]}}
    }
    m = compute_metrics_for_accn(facts, "0000999999-25-000001", "10-Q", report_date="2025-09-28")
    assert m["revenue"] == 35_000_000_000.0


def test_compute_metrics_report_date_fallback_inactive_without_report_date():
    """Without report_date kwarg, unknown accn still yields None (no spurious matches)."""
    facts = {
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            {"accn": "0000000-other-accn", "end": "2025-09-28", "val": 35_000_000_000,
             "form": "10-Q", "fp": "Q3", "fy": 2025, "filed": "2025-10-30"},
        ]}}
    }
    m = compute_metrics_for_accn(facts, "0000999999-25-000001", "10-Q")
    assert m["revenue"] is None


def test_compute_metrics_report_date_fallback_ignores_dates_outside_window():
    """A 16-day gap between report_date and entry 'end' does not trigger fallback."""
    facts = {
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            {"accn": "0000000-other-accn", "end": "2025-09-01", "val": 35_000_000_000,
             "form": "10-Q", "fp": "Q3", "fy": 2025, "filed": "2025-10-30"},
        ]}}
    }
    # 27 days gap: 2025-09-28 vs 2025-09-01 → outside ±15-day window
    m = compute_metrics_for_accn(facts, "0000999999-25-000001", "10-Q", report_date="2025-09-28")
    assert m["revenue"] is None


# ── XBRL context selection / standalone-quarter normalization ────────────────

def _fact(accn, start, end, val, *, fy=2026, fp="Q2", form="10-Q", frame=None):
    row = {
        "accn": accn,
        "start": start,
        "end": end,
        "val": val,
        "form": form,
        "fp": fp,
        "fy": fy,
        "filed": "2026-08-20",
    }
    if frame:
        row["frame"] = frame
    return row


def test_10q_selects_current_three_month_context_not_comparative_or_ytd():
    """One accession can contain prior-year, current-quarter and current-YTD facts."""
    current_accn = "0001045810-26-000099"
    facts = {
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            _fact(current_accn, "2025-04-28", "2025-07-27", 30_000, frame="CY2025Q2"),
            _fact(current_accn, "2026-01-26", "2026-07-26", 145_000),  # six-month YTD
            _fact(current_accn, "2026-04-27", "2026-07-26", 81_000, frame="CY2026Q2"),
        ]}},
        "GrossProfit": {"units": {"USD": [
            _fact(current_accn, "2026-01-26", "2026-07-26", 100_000),
            _fact(current_accn, "2026-04-27", "2026-07-26", 60_750, frame="CY2026Q2"),
        ]}},
        "OperatingIncomeLoss": {"units": {"USD": [
            _fact(current_accn, "2026-01-26", "2026-07-26", 80_000),
            _fact(current_accn, "2026-04-27", "2026-07-26", 48_600, frame="CY2026Q2"),
        ]}},
        "EarningsPerShareDiluted": {"units": {"USD/shares": [
            _fact(current_accn, "2026-01-26", "2026-07-26", 4.20),
            _fact(current_accn, "2026-04-27", "2026-07-26", 2.39, frame="CY2026Q2"),
        ]}},
    }

    result = compute_metrics_for_accn(
        facts, current_accn, "10-Q", report_date="2026-07-26"
    )

    assert result["revenue"] == 81_000
    assert result["eps"] == 2.39
    assert result["gross_margin"] == pytest.approx(0.75)
    assert result["operating_margin"] == pytest.approx(0.60)


def test_10q_cash_flow_values_are_deaccumulated_to_standalone_quarter():
    """Q2/Q3 cash-flow facts are YTD and must be reduced by the prior YTD value."""
    q1_accn = "0001045810-26-000050"
    q2_accn = "0001045810-26-000099"
    facts = {
        "NetCashProvidedByUsedInOperatingActivities": {"units": {"USD": [
            _fact(q1_accn, "2026-01-26", "2026-04-26", 30_000, fp="Q1"),
            _fact(q2_accn, "2026-01-26", "2026-07-26", 70_000, fp="Q2"),
        ]}},
        "PaymentsToAcquirePropertyPlantAndEquipment": {"units": {"USD": [
            _fact(q1_accn, "2026-01-26", "2026-04-26", 2_000, fp="Q1"),
            _fact(q2_accn, "2026-01-26", "2026-07-26", 5_500, fp="Q2"),
        ]}},
        "PaymentsForRepurchaseOfCommonStock": {"units": {"USD": [
            _fact(q1_accn, "2026-01-26", "2026-04-26", 10_000, fp="Q1"),
            _fact(q2_accn, "2026-01-26", "2026-07-26", 24_000, fp="Q2"),
        ]}},
        # A small duration fact establishes the accession's fiscal year/period.
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            _fact(q2_accn, "2026-04-27", "2026-07-26", 81_000, fp="Q2", frame="CY2026Q2"),
        ]}},
    }

    result = compute_metrics_for_accn(facts, q2_accn, "10-Q", report_date="2026-07-26")

    assert result["capex"] == 3_500
    assert result["free_cash_flow"] == 36_500
    assert result["buybacks"] == 14_000
    assert result["period_basis"] == "quarter"


def test_10k_selects_current_fiscal_year_context_not_comparative():
    accn = "0001045810-27-000010"
    facts = {
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            _fact(accn, "2025-01-27", "2026-01-25", 210_000, fy=2027, fp="FY", form="10-K"),
            _fact(accn, "2026-01-26", "2027-01-31", 350_000, fy=2027, fp="FY", form="10-K"),
        ]}},
    }

    result = compute_metrics_for_accn(facts, accn, "10-K", report_date="2027-01-31")

    assert result["revenue"] == 350_000
    assert result["period"] == "FY2027"
    assert result["period_basis"] == "annual"


def test_10q_rejects_ytd_revenue_when_standalone_quarter_is_missing():
    """A six-month flow must never be presented as the standalone Q2 value."""
    accn = "0001045810-26-000099"
    facts = {
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            _fact(accn, "2026-01-26", "2026-07-26", 145_000, fp="Q2"),
        ]}},
    }

    result = compute_metrics_for_accn(
        facts, accn, "10-Q", report_date="2026-07-26", filing_date="2026-08-20"
    )

    assert result["revenue"] is None
    assert "revenue:duration_mismatch" in result["quality_warnings"]
    assert result["data_quality_status"] == "CHECK_REQUIRED"


def test_balance_sheet_metric_rejects_duration_context():
    """Debt/equity require an instant fact, not a start/end duration fact."""
    accn = "0001045810-26-000099"
    facts = {
        "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            _fact(accn, "2026-04-27", "2026-07-26", 81_000, fp="Q2"),
        ]}},
        "LongTermDebt": {"units": {"USD": [
            _fact(accn, "2026-04-27", "2026-07-26", 25_000, fp="Q2"),
        ]}},
    }

    result = compute_metrics_for_accn(
        facts, accn, "10-Q", report_date="2026-07-26", filing_date="2026-08-20"
    )

    assert result["total_debt"] is None
    assert "total_debt:instant_context_missing" in result["quality_warnings"]
    assert result["data_quality_status"] == "CHECK_REQUIRED"