scripts/seed_expand_statewide.py

"""Phase-2 seed expansion: statewide CA ocean species (esp. Southern + inverts).

Every value verified against the cited CDFW page or CCR Title 14 during the
2026-06-15 research pass. Species without a curated rule card are detected via
the fact alias index, so facts alone make them answerable.
"""

import json
from pathlib import Path

FACTS_PATH = Path("data/source-cache/seed/facts.json")

SOUTH = "cdfw_region_southern"
SOUTH_URL = "https://wildlife.ca.gov/Fishing/Ocean/Regulations/Fishing-Map/Southern"
OCEAN = "cdfw_ocean_regulations"
GENERAL_URL = "https://wildlife.ca.gov/Fishing/Ocean/Regulations/Sport-Fishing/General-Ocean-Fishing-Regs"
INVERT_URL = "https://wildlife.ca.gov/Fishing/Ocean/Regulations/Sport-Fishing/Invertebrate-Fishing-Regs"
NOW = "2026-06-15T12:00:00+00:00"

GENERAL_BAG = "10"
GENERAL_BAG_UNITS = (
    "fish of any one species per day, within the general limit of 20 finfish in "
    "combination of all species (no species-specific minimum size limit; CCR T14 27.60)"
)
NO_TAKE = "Take and possession prohibited in California ocean waters; may not be taken or possessed."


def f(id, species, display, fact_type, value, *, units=None, source_id, source_url,
      snippet, aliases=(), region_key=None, eff_start=None, eff_end=None):
    entry = {
        "id": id, "species_or_category": species, "display_name": display,
        "aliases": list(aliases), "area_or_scope": "California ocean sport fishing regulations",
        "fact_type": fact_type, "value": value, "units": units,
        "source_id": source_id, "source_url": source_url, "retrieved_at": NOW,
        "supporting_chunk_id": "", "supporting_snippet": snippet, "validation_set": False,
    }
    if region_key:
        entry["region_key"] = region_key
    if eff_start:
        entry["effective_start"] = eff_start
    if eff_end:
        entry["effective_end"] = eff_end
    return entry


def general_limit(key, display, aliases):
    return f(f"{key}_bag_limit", key, display, "bag_limit", GENERAL_BAG, units=GENERAL_BAG_UNITS,
             source_id=OCEAN, source_url=GENERAL_URL, aliases=aliases,
             snippet="No more than 20 finfish in combination of all species, with not more than 10 of any one species, may be taken or possessed (CCR T14 27.60).")


def no_limit(key, display, aliases):
    return f(f"{key}_bag_limit", key, display, "bag_limit", "no daily bag limit", units="(CCR T14 27.60(b))",
             source_id=OCEAN, source_url=GENERAL_URL, aliases=aliases,
             snippet="There is no limit on the following species: anchovy, jacksmelt, topsmelt, Pacific butterfish (pompano), queenfish, sanddabs, skipjack, jack mackerel, Pacific mackerel, ... Pacific sardine ... (CCR T14 27.60(b)).")


def no_take(key, display, aliases, source_url=GENERAL_URL, source_id=OCEAN, snippet=None):
    return f(f"{key}_closure", key, display, "closure", NO_TAKE, source_id=source_id, source_url=source_url,
             aliases=aliases, snippet=snippet or f"{display}: may not be taken or possessed.")


NEW = [
    # --- Southern California bass complex (CCR 28.30 / Southern page) ---
    f("kelp_bass_min_size", "kelp_bass", "Kelp bass", "minimum_size", "14", units="inches total length",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["calico bass", "calico", "kelp bass"],
      snippet="Kelp bass, barred sand bass and spotted sand bass: the minimum size limit is 14 inches total length."),
    f("kelp_bass_bag_limit", "kelp_bass", "Kelp bass", "bag_limit", "5",
      units="fish in any combination of kelp, barred sand, and spotted sand bass per day",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["calico bass"],
      snippet="Five fish in any combination of species (kelp bass, barred sand bass, spotted sand bass)."),
    f("barred_sand_bass_min_size", "barred_sand_bass", "Barred sand bass", "minimum_size", "14", units="inches total length",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["sand bass"],
      snippet="Kelp bass, barred sand bass and spotted sand bass: the minimum size limit is 14 inches total length."),
    f("barred_sand_bass_bag_limit", "barred_sand_bass", "Barred sand bass", "bag_limit", "5",
      units="fish in any combination of kelp/barred sand/spotted sand bass, of which no more than 4 may be barred sand bass",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["sand bass"],
      snippet="Five fish in any combination of species, except no more than 4 barred sand bass may be taken. The minimum size limit is 14 inches."),
    f("spotted_sand_bass_min_size", "spotted_sand_bass", "Spotted sand bass", "minimum_size", "14", units="inches total length",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["spotted bay bass", "spotty"],
      snippet="Kelp bass, barred sand bass and spotted sand bass: the minimum size limit is 14 inches total length."),
    f("spotted_sand_bass_bag_limit", "spotted_sand_bass", "Spotted sand bass", "bag_limit", "5",
      units="fish in any combination of kelp/barred sand/spotted sand bass per day",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["spotted bay bass"],
      snippet="Five fish in any combination of species (kelp bass, barred sand bass, spotted sand bass)."),
    # --- California sheephead (Southern page) ---
    f("california_sheephead_min_size", "california_sheephead", "California sheephead", "minimum_size", "12", units="inches total length",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["sheephead", "sheepshead"],
      snippet="California sheephead: 2 fish, with a minimum size limit of 12 inches total length."),
    f("california_sheephead_bag_limit", "california_sheephead", "California sheephead", "bag_limit", "2", units="fish per day",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["sheephead"],
      snippet="California sheephead: 2 fish, with a minimum size limit of 12 inches total length."),
    # --- California barracuda (CCR 28.25) ---
    f("california_barracuda_min_size", "california_barracuda", "California barracuda", "minimum_size", "28",
      units="inches total length (or 17 inches alternate length)",
      source_id=OCEAN, source_url=GENERAL_URL, aliases=["barracuda", "pacific barracuda"],
      snippet="California barracuda minimum size: twenty-eight inches total length or seventeen inches alternate length (CCR T14 28.25)."),
    # --- Pacific bonito (CCR 28.32) ---
    f("pacific_bonito_min_size", "pacific_bonito", "Pacific bonito", "minimum_size", "24",
      units="inches fork length or 5 pounds, except up to 5 smaller fish may be taken",
      source_id=OCEAN, source_url=GENERAL_URL, aliases=["bonito"],
      snippet="Pacific bonito minimum size: twenty-four inches fork length or five pounds, except that five fish less than twenty-four inches or five pounds may be taken (CCR T14 28.32)."),
    f("pacific_bonito_bag_limit", "pacific_bonito", "Pacific bonito", "bag_limit", "10", units="fish per day",
      source_id=OCEAN, source_url=GENERAL_URL, aliases=["bonito"],
      snippet="Pacific bonito limit: ten (CCR T14 28.32)."),
    # --- California yellowtail (Southern page) ---
    f("california_yellowtail_min_size", "california_yellowtail", "California yellowtail", "minimum_size", "24",
      units="inches fork length, except up to 5 smaller fish may be taken",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["yellowtail"],
      snippet="California yellowtail: the minimum size limit is 24 inches fork length, except that up to five fish less than 24 inches fork length may be taken."),
    f("california_yellowtail_bag_limit", "california_yellowtail", "California yellowtail", "bag_limit", "10", units="fish per day",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["yellowtail"],
      snippet="California yellowtail: ten fish."),
    # --- Ocean whitefish (Southern page) ---
    f("ocean_whitefish_min_size", "ocean_whitefish", "Ocean whitefish", "minimum_size", "no minimum size limit",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["whitefish"],
      snippet="Ocean whitefish: 10 fish within the general daily bag limit of 20 fish total, with no minimum size limit."),
    f("ocean_whitefish_bag_limit", "ocean_whitefish", "Ocean whitefish", "bag_limit", "10",
      units="fish within the general daily bag limit of 20 finfish total",
      source_id=SOUTH, source_url=SOUTH_URL, aliases=["whitefish"],
      snippet="Ocean whitefish: 10 fish within the general daily bag limit of 20 fish total."),
    # --- California grunion (CCR 28.00) ---
    f("california_grunion_season", "california_grunion", "California grunion", "season",
      "open to take by hand only, except closed (no take) from April 1 through June 30.",
      source_id=OCEAN, source_url=GENERAL_URL, aliases=["grunion"], eff_start="2026-04-01", eff_end="2026-06-30",
      snippet="It shall be unlawful to take grunion from April 1 through June 30 (CCR T14 28.00)."),
    f("california_grunion_bag_limit", "california_grunion", "California grunion", "bag_limit", "30", units="fish per day",
      source_id=OCEAN, source_url=GENERAL_URL, aliases=["grunion"],
      snippet="California grunion limit: 30 (CCR T14 28.00)."),
    # --- Giant (black) sea bass: no-take off California (CCR 28.10) ---
    no_take("giant_sea_bass", "Giant sea bass", ["black sea bass", "giant black sea bass"],
            snippet="Giant sea bass may not be taken off California; fish taken incidental to other fishing must be immediately returned to the water (CCR T14 28.10)."),
    # --- No-limit baitfish/pelagics (CCR 27.60(b)) ---
    no_limit("northern_anchovy", "Northern anchovy", ["anchovy", "anchovies"]),
    no_limit("pacific_sardine", "Pacific sardine", ["sardine", "sardines"]),
    no_limit("pacific_mackerel", "Pacific mackerel", ["mackerel", "spanish mackerel"]),
    no_limit("jack_mackerel", "Jack mackerel", ["jack mackerel"]),
    no_limit("queenfish", "Queenfish", ["queenfish", "herring queenfish"]),
    no_limit("pacific_butterfish", "Pacific butterfish (pompano)", ["butterfish", "pompano", "pacific pompano"]),
    # --- General-limit nearshore SoCal finfish (CCR 27.60) ---
    general_limit("white_croaker", "White croaker", ["white croaker", "kingfish", "tomcod"]),
    general_limit("yellowfin_croaker", "Yellowfin croaker", ["yellowfin croaker"]),
    general_limit("spotfin_croaker", "Spotfin croaker", ["spotfin croaker"]),
    general_limit("california_corbina", "California corbina", ["corbina", "corvina"]),
    general_limit("opaleye", "Opaleye", ["opaleye", "opal eye"]),
    general_limit("halfmoon", "Halfmoon", ["halfmoon", "blue perch", "catalina blue perch"]),
    general_limit("sargo", "Sargo", ["sargo", "china croaker"]),
    # --- Sharks / rays / skates ---
    general_limit("spiny_dogfish", "Spiny dogfish", ["spiny dogfish", "dogfish", "mud shark"]),
    general_limit("skates", "Skates", ["skate", "skates", "big skate", "longnose skate"]),
    general_limit("bat_ray", "Bat ray", ["bat ray", "bat stingray"]),
    general_limit("shovelnose_guitarfish", "Shovelnose guitarfish", ["guitarfish", "shovelnose guitarfish", "shovelnose shark"]),
    general_limit("thornback_ray", "Thornback ray", ["thornback", "thornback ray"]),
    general_limit("round_stingray", "Round stingray", ["round stingray", "stingray", "round ray"]),
    no_take("white_shark", "White shark (great white)", ["great white", "white shark", "great white shark"],
            source_id=SOUTH, source_url=SOUTH_URL,
            snippet="White sharks may not be taken or possessed at any time (CCR T14 28.06; CDFW Southern region)."),
    no_take("gulf_grouper", "Gulf grouper", ["gulf grouper"],
            snippet="Gulf grouper may not be taken or possessed (CCR T14 28.12)."),
    no_take("broomtail_grouper", "Broomtail grouper", ["broomtail grouper"],
            snippet="Broomtail grouper may not be taken or possessed (CCR T14 28.12)."),
    no_take("basking_shark", "Basking shark", ["basking shark"],
            snippet="Basking sharks are prohibited from general retention in the U.S. West Coast HMS fishery (50 CFR 660.705)."),
    no_take("megamouth_shark", "Megamouth shark", ["megamouth", "megamouth shark"],
            snippet="Megamouth sharks are prohibited from general retention in the U.S. West Coast HMS fishery (50 CFR 660.705)."),
    # --- Invertebrates ---
    f("california_spiny_lobster_min_size", "california_spiny_lobster", "California spiny lobster", "minimum_size",
      "3¼", units="inches carapace (straight line on the mid-line of the back, rear of eye socket to rear of body shell)",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["spiny lobster"],
      snippet="California spiny lobster minimum size: 3 and 1/4 inches measured on the mid-line of the back from the rear edge of the eye socket to the rear edge of the body shell (CCR T14 29.90)."),
    f("california_spiny_lobster_bag_limit", "california_spiny_lobster", "California spiny lobster", "bag_limit", "7", units="lobster per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["spiny lobster"],
      snippet="California spiny lobster daily bag limit: seven (CCR T14 29.90)."),
    f("california_spiny_lobster_season", "california_spiny_lobster", "California spiny lobster", "season",
      "open from 6:00 p.m. on the Friday preceding the first Wednesday in October through the first Wednesday after the 15th of March.",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["spiny lobster"],
      snippet="Open season: from 6:00 p.m. on the Friday before the first Wednesday in October through the first Wednesday after the 15th of March (CCR T14 29.90)."),
    f("rock_scallop_bag_limit", "rock_scallop", "Rock scallop", "bag_limit", "10", units="scallops per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["rock scallop", "scallop", "scallops"],
      snippet="Rock scallop limit: ten (CCR T14 29.60)."),
    no_take("speckled_scallop", "Speckled (bay) scallop", ["bay scallop", "speckled scallop"],
            source_id=OCEAN, source_url=INVERT_URL,
            snippet="Speckled (bay) scallops may not be taken or possessed (CCR T14 29.65)."),
    f("market_squid_bag_limit", "market_squid", "Market squid", "bag_limit", "no daily bag limit", units="(may be taken with hand-held dip nets)",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["squid", "calamari", "market squid"],
      snippet="Squid may be taken with hand-held dip nets. There is no limit (CCR T14 29.70)."),
    f("kellets_whelk_bag_limit", "kellets_whelk", "Kellet's whelk", "bag_limit", "35", units="whelks per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["whelk", "kellets whelk", "kellet's whelk"],
      snippet="Kellet's whelk: 35 whelks; season July 1 through the first Wednesday after the 15th of March (CCR T14 29.17, 29.05)."),
    f("octopus_bag_limit", "octopus", "Octopus", "bag_limit", "35", units="octopus per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["octopus", "octopuses", "octopi"],
      snippet="Octopus: 35 octopus, year round (CCR T14 29.05)."),
    f("native_oyster_bag_limit", "native_oyster", "Native oyster", "bag_limit", "35", units="oysters per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["oyster", "oysters", "native oyster"],
      snippet="Oysters, all: 35 oysters, year round (CCR T14 29.05)."),
    f("ghost_shrimp_bag_limit", "ghost_shrimp", "Ghost shrimp and blue mud shrimp", "bag_limit", "50", units="in combination per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["ghost shrimp", "mud shrimp", "blue mud shrimp"],
      snippet="Ghost shrimp and blue mud shrimp limit: fifty in combination (CCR T14 29.87)."),
    f("coonstripe_shrimp_bag_limit", "coonstripe_shrimp", "Coonstripe shrimp", "bag_limit", "20",
      units="pounds in the shell, heads on, per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["coonstripe shrimp"],
      snippet="Coonstripe shrimp: twenty pounds (in the shell, heads on) (CCR T14 29.88)."),
    f("sand_crab_bag_limit", "sand_crab", "Sand crab (mole crab)", "bag_limit", "50", units="per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["sand crab", "mole crab", "sand crabs"],
      snippet="Sand crabs (Emerita analoga): fifty (CCR T14 29.85)."),
    # --- Distinct named clams (own CCR sections) ---
    f("gaper_clam_bag_limit", "gaper_clam", "Gaper clam (horseneck clam)", "bag_limit", "10",
      units="of each species per day (Humboldt Bay: 50 in combination, max 25 gaper; Elkhorn Slough: 12 in combination)",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["gaper clam", "horseneck clam", "horse clam"],
      snippet="Gaper/horseneck clams: limit ten of each species, except in Humboldt Bay fifty in combination (no more than 25 gaper) and in Elkhorn Slough twelve in combination (CCR T14 29.25)."),
    f("geoduck_clam_bag_limit", "geoduck_clam", "Geoduck clam", "bag_limit", "3", units="clams per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["geoduck", "geoduck clam"],
      snippet="Geoduck clams: limit three; the first three dug must be retained as the bag limit (CCR T14 29.30)."),
    f("pismo_clam_min_size", "pismo_clam", "Pismo clam", "minimum_size", "5", units="inches greatest diameter",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["pismo clam"],
      snippet="Pismo clams: minimum size five inches in greatest diameter; limit ten (CCR T14 29.40)."),
    f("pismo_clam_bag_limit", "pismo_clam", "Pismo clam", "bag_limit", "10", units="clams per day",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["pismo clam"],
      snippet="Pismo clams: limit ten (CCR T14 29.40)."),
    f("razor_clam_bag_limit", "razor_clam", "Razor clam", "bag_limit", "20",
      units="clams per day (the first 20 dug must be retained regardless of size or condition)",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["razor clam", "razor clams"],
      snippet="Razor clams: limit twenty; the first twenty dug must be retained as the bag limit (CCR T14 29.45)."),
    f("littleneck_clam_bag_limit", "littleneck_clam", "Littleneck, cockle, and softshell clams", "bag_limit", "50",
      units="in combination per day (minimum size 1½ inches, except no size limit for soft-shell clams)",
      source_id=OCEAN, source_url=INVERT_URL, aliases=["littleneck clam", "cockle", "softshell clam", "soft-shell clam", "quahog", "chione"],
      snippet="Littleneck clams, soft-shell clams, chiones, northern quahogs and cockles: limit fifty in combination; minimum size one and one-half inches, except no size limit for soft-shell clams (CCR T14 29.35)."),
]


def main():
    data = json.loads(FACTS_PATH.read_text())
    by_id = {f["id"]: i for i, f in enumerate(data["structured_facts"])}
    added = updated = 0
    for entry in NEW:
        if entry["id"] in by_id:
            data["structured_facts"][by_id[entry["id"]]] = entry
            updated += 1
        else:
            data["structured_facts"].append(entry)
            added += 1
    FACTS_PATH.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n")
    species = {f["species_or_category"] for f in data["structured_facts"]}
    print(f"added {added}, updated {updated}; total facts {len(data['structured_facts'])}, distinct species {len(species)}")


if __name__ == "__main__":
    main()