scripts/extract_poucher_tiers.py

#!/usr/bin/env python3
"""Extract Poucher evaporation coefficient tier assignments from the literature pages.

Scans all 3013 pages for "Odour classification" sections, parses the
Top/Middle/Basic notes tables, and maps each material to its tier
based on Poucher's coefficient number system:
  Top notes:   coefficient 1-14
  Middle notes: coefficient 15-60
  Base notes:   coefficient 61-100
"""
import json
import re
from pathlib import Path
from collections import defaultdict

DATA = Path("data")


def load_pages() -> list[dict]:
    """Load all literature pages, extracting text and source."""
    pages = []
    with open(DATA / "literature_flat" / "literature_pages.jsonl") as f:
        for line in f:
            record = json.loads(line).get("record", "")
            if isinstance(record, str):
                try:
                    rec = json.loads(record)
                except json.JSONDecodeError:
                    continue
            else:
                rec = record
            text = rec.get("text", "")
            source = rec.get("source", "")
            page_num = rec.get("page", 0)
            if text and "Poucher" in source:
                pages.append({"page": page_num, "text": text, "source": source})
    return pages


def parse_odour_classification(text: str) -> list[dict]:
    """Parse a single page's Odour Classification section.

    Structure observed:
        Odour classification
        Top notes
        1. Benzyl acetate
        Linalol
        Phenyl ethyl acetate
        2. Rosewood
        ...
        Middle notes
        15. Acet anisol
        Heliotropin
        21. Anisic aldehyde
        Ionone alpha
        ...
        Basic notes
        65. Cinnamic alcohol
        77. Methyl naphthyl ketone
        ...

    Key insight: materials listed after a numbered entry and before the next number
    belong to the same coefficient group.
    """
    results = []

    # Find the "Odour classification" section
    oc_match = re.search(r'[Oo]dour\s+[Cc]lassification', text)
    if not oc_match:
        return results

    section = text[oc_match.start():]

    # Find tier section boundaries
    tier_keywords = [
        (r'(?:^|\n)\s*Top\s+notes?\s*(?:\n|$)', 'top'),
        (r'(?:^|\n)\s*Middle\s+notes?\s*(?:\n|$)', 'mid'),
        (r'(?:^|\n)\s*(?:Basic|Base)\s+notes?\s*(?:\n|$)', 'base'),
    ]

    tier_positions = []
    for pattern, tier in tier_keywords:
        for m in re.finditer(pattern, section):
            tier_positions.append((m.start(), m.end(), tier))

    tier_positions.sort()

    for i, (start, header_end, tier) in enumerate(tier_positions):
        # Section text for this tier
        end = tier_positions[i + 1][0] if i + 1 < len(tier_positions) else min(len(section), start + 2000)
        tier_text = section[header_end:end]

        # Parse lines: numbered entries set the coefficient, unnumbered lines
        # that follow are materials at the same coefficient
        current_coeff = None
        lines = tier_text.strip().split('\n')

        for line in lines:
            line = line.strip()
            if not line:
                continue

            # Check if this line starts with a number (coefficient)
            num_match = re.match(r'^(\d{1,3})\.\s*(.+)', line)
            if num_match:
                current_coeff = int(num_match.group(1))
                material_name = num_match.group(2).strip()
                if material_name and len(material_name) > 1:
                    results.append({
                        "coefficient": current_coeff,
                        "tier": tier,
                        "name": material_name,
                    })
            else:
                # This is a continuation material — belongs to the current coefficient
                # Filter out prose (sentences) and formula-like entries
                if (current_coeff is not None
                    and len(line) > 1
                    and len(line) < 50
                    and not line[0].isdigit()  # not a formula amount
                    and not line.startswith('Compounding')
                    and not line.startswith('Soap')
                    and not line.startswith('page')
                    and not '.' in line[:5]  # not a page number
                    and line[0].isupper()  # material names start with capital
                    and not any(w in line.lower() for w in ['notes', 'classification', 'perfum', 'soap', 'chapter'])):
                    results.append({
                        "coefficient": current_coeff,
                        "tier": tier,
                        "name": line,
                    })

    return results


# Extended name → CAS mapping based on Poucher materials
POUCHER_NAME_TO_CAS = {
    # Top notes (coeff 1-14)
    "benzyl acetate": "140-11-4",
    "linalol": "78-70-6",
    "linalool": "78-70-6",
    "paracresyl acetate": "140-39-6",
    "p-cresyl acetate": "140-39-6",
    "benzaldehyde": "100-52-7",
    "almonds": "100-52-7",
    "phenyl ethyl acetate": "103-45-7",
    "phenylethyl acetate": "103-45-7",
    "benzyl cinnamate": "103-41-3",
    "terpineol": "8000-41-7",
    "alpha-terpineol": "98-55-5",
    "citronellol": "106-22-9",
    "lavender": "8000-28-0",
    "bergamot": "8007-75-8",
    "geraniol": "106-24-1",
    "geraniol java": "106-24-1",
    "amyl salicylate": "2050-08-0",
    "lemon": "8008-56-8",
    "limes": "8008-26-2",
    "sweet orange": "8008-57-9",
    "cedarwood": "8000-27-9",
    "rosewood": "8015-77-8",
    "bois de rose": "8015-77-8",
    "linalyl acetate": "115-95-7",
    "methyl cinnamate": "103-26-4",
    "cananga": "68606-83-7",
    "ylang": "8006-81-3",
    "ylang-ylang": "8006-81-3",
    "lavandin": "8022-15-7",
    "petitgrain para": "8014-17-3",
    "petitgrain": "8014-17-3",
    "spike lavender": "8022-09-9",
    "methyl salicylate": "119-36-8",
    "methyl benzoate": "93-58-3",
    "methyl anthranilate": "134-20-3",
    "citronella ceylon": "8000-29-1",
    "citronella java": "91771-61-8",
    "phenyl ethyl alcohol": "60-12-8",
    "phenylethyl alcohol": "60-12-8",
    "bromstyrole": "103-64-0",
    "cumic aldehyde": "122-03-2",
    "cuminic aldehyde": "122-03-2",
    "methyl octine carbonate": "111-12-6",
    "methyl heptine carbonate": "111-12-6",
    "dimethyl benzyl carbinol": "100-86-7",
    "nonyl aldehyde": "124-19-6",
    "nonyl aldehyde": "124-19-6",
    "decaldehyde": "112-31-2",
    "decyl aldehyde": "112-31-2",
    "methyl acetophenone": "122-00-9",
    "diphenyl oxide": "101-84-8",
    "diphenyl ether": "101-84-8",
    "carrot seed": "8015-88-1",
    "methyl ionone": "1335-46-2",
    "orris concrete": "8023-85-4",
    "orris": "8023-85-4",
    "mimosa absolute": "8023-87-6",
    "reseda absolute": "8022-62-6",
    "terpinyl acetate": "8007-35-0",
    "linalyl benzoate": "126-64-7",
    "phenyl ethyl benzoate": "94-47-3",
    "citronellyl formate": "105-85-1",
    "citronellyl acetate": "150-84-5",
    "geranyl acetate": "105-87-3",
    "linalyl propionate": "144-39-8",
    "nerol": "106-25-2",
    "neryl acetate": "141-12-8",
    "rhodinol": "68127-65-1",
    "palmarosa": "8014-19-5",
    "geraniol palmarosa": "8014-19-5",
    "sassafras": "94-59-7",

    # Middle notes (coeff 15-60)
    "acet anisol": "104-21-2",
    "heliotropin": "120-57-0",
    "piperonal": "120-57-0",
    "eugenol": "97-53-0",
    "clove": "8000-34-8",
    "cinnamyl acetate": "103-54-8",
    "anisic aldehyde": "123-11-5",
    "anisaldehyde": "123-11-5",
    "ionone alpha": "127-41-3",
    "alpha ionone": "127-41-3",
    "ionone beta": "79-77-6",
    "beta ionone": "79-77-6",
    "ionone": "127-41-3",
    "clary sage": "8016-63-5",
    "verbena": "8024-12-6",
    "methyl anthranilate": "134-20-3",
    "dimethyl hydroquinone": "615-90-3",
    "geranium bourbon": "8000-46-2",
    "geranium african": "8000-46-2",
    "geranium": "8000-46-2",
    "orange flower absolute": "8016-38-0",
    "neroli": "8016-38-0",
    "jasmin absolute": "8024-43-9",
    "rose absolute": "8007-01-0",
    "rose otto": "8007-01-0",
    "rose": "8007-01-0",
    "laurinic aldehyde": "112-54-9",
    "dodecyl aldehyde": "112-54-9",
    "lauryl aldehyde": "112-54-9",
    "cinnamon leaf": "8015-91-6",
    "cinnamon": "8015-91-6",
    "cassia": "8015-96-1",
    "ethyl cinnamate": "103-36-6",
    "cassia oil": "8015-96-1",
    "serpolet": "84012-66-8",
    "melissa": "8014-71-9",
    "calamus": "8015-79-2",
    "marjoram": "8015-01-0",
    "angelica seed": "8015-64-3",
    "bornyl acetate": "76-49-3",
    "phenyl ethyl iso-butyrate": "103-48-0",
    "phenyl ethyl butyrate": "103-52-6",
    "phenoxy ethyl iso-butyrate": "103-60-6",
    "phenoxyethyl iso-butyrate": "103-60-6",
    "phenoxyethyl isobutyrate": "103-60-6",
    "phenyl methyl carbinyl acetate": "93-92-5",
    "citral": "5392-40-1",
    "gingergrass": "8023-70-5",
    "methyl nonyl acetaldehyde": "110-41-8",
    "aldehyde c-12 mna": "110-41-8",
    "cinnamyl formate": "104-65-4",
    "guaiac wood": "8016-23-7",
    "phenoxyethyl alcohol": "622-08-2",

    # Base notes (coeff 61-100)
    "cinnamic alcohol": "104-54-1",
    "cinnamyl alcohol": "104-54-1",
    "methyl naphthyl ketone": "94-90-6",
    "methyl naphthal ketone": "94-90-6",
 "civet absolute": "68991-27-5",
    "hydroxy citronellal": "107-75-5",
    "hydroxycitronellal": "107-75-5",
    "phenyl acetaldehyde": "122-78-1",
    "phenyl acetic acid": "103-82-2",
    "phenylacetic acid": "103-82-2",
    "ethyl methyl phenyl glycidate": "77-83-8",
    "rhodinyl acetate": "141-14-0",
    "rhodinyl formate": "83-54-5",
    "undecalactone": "104-67-6",
    "gamma-undecalactone": "104-67-6",
    "amyl cinnamic aldehyde": "122-40-7",
    "amyl cinnamaldehyde": "122-40-7",
    "benzoin resin": "9000-72-0",
    "benzoin": "9000-72-0",
    "coumarin": "91-64-5",
    "musk xylene": "81-15-2",
    "musk ketone": "81-14-1",
    "peru balsam": "8007-00-9",
    "tolu balsam": "9000-64-0",
    "styrax resin": "8024-01-9",
    "styrax": "8024-01-9",
    "vanillin": "121-33-5",
    "vetivert": "8016-96-4",
    "vetiver": "8016-96-4",
    "patchouli": "8014-09-3",
    "oakmoss": "9000-50-6",
    "labdanum resin": "8016-73-1",
    "labdanum": "8016-73-1",
    "castoreum absolute": "8023-83-4",
    "castoreum": "8023-83-4",
    "santal": "8006-87-9",
    "sandalwood": "8006-87-9",
    "cedar": "8000-27-9",
    "opoponax resin": "8021-15-0",
    "myrrh resin": "8023-82-3",
    "myrrh": "8023-82-3",
    "ambergris": "8038-65-1",
    "cassie absolute": "8015-61-0",
    "cassie absolute farnesiana": "8015-61-0",
    "tuberose absolute": "8024-05-2",
    "phenyl acetic aldehyde": "122-78-1",
    "opoponax oil": "8021-15-0",
    "olibanum resin": "8050-07-5",
    "olibanum": "8050-07-5",
    "benzyl salicylate": "118-58-1",
    "iso-eugenol": "97-54-1",
    "isoeugenol": "97-54-1",
    "benzyl iso-eugenol": "93-26-3",
    "iso-butyl salicylate": "87-19-4",
    "geranyl benzoate": "94-48-4",
    "methyl salicylate": "119-36-8",
    "linalyl salicylate": "7149-28-2",
    "benzophenone": "119-61-9",
    "phenyl carbonate": "135-20-6",
    "ethyl decine carbonate": "10031-93-5",
    "decanal": "112-31-2",
    "octyl aldehyde": "124-13-0",
    "estragnol": "8015-79-2",
    "estragon": "8015-79-2",
    "iso-butyl quinoline": "93-19-6",
    "isobutyl quinoline": "93-19-6",
    "cinnamic aldehyde": "104-55-2",
    "cinnamaldehyde": "104-55-2",
    "trichlor phenyl methyl carbinyl acetate": "90-17-5",
    "acetophenone": "98-86-2",
    "phenyl acetaldehyde dimethyl acetal": "101-48-8",
    "phenyl ethyl dimethyl acetal": "67674-46-8",
    "paracresyl phenylacetate": "101-94-0",
    "iso-butyl phenylacetate": "102-13-6",
    "paracresyl methyl ether": "104-93-8",
    "benzylidene acetone": "122-57-6",
    # Additional materials found in unmatched list
    "benzyl alcohol": "100-51-6",
    "benzyl benzoate": "120-51-4",
    "benzyl formate": "104-57-4",
    "benzyl propionate": "122-63-4",
    "benzyl phenylacetate": "102-16-9",
    "benzyl iso-butyrate": "103-09-3",
    "benzyl isoeugenol": "93-26-3",
    "anisyl acetate": "104-21-2",
    "anisic alcohol": "105-13-5",
    "anisyl alcohol": "105-13-5",
    "bay": "8006-78-8",
    "ambrette seed": "8015-65-4",
    "angelica root": "8015-64-3",
    "basilic": "8015-73-4",
    "basil": "8015-73-4",
    "amyl cinnamate": "3487-99-8",
    "amyl oxyiso": "68966-86-9",
    "acet eugenol": "93-28-7",
    "eugenyl acetate": "93-28-7",
    "acetyl iso-eugenol": "93-29-6",
    "acetiso-eugenol": "93-29-6",
    "phenoxyethyl alcohol": "622-08-2",
    "phenyl ethyl alcohol": "60-12-8",
    "methyl benzoate": "93-58-3",
    "ethyl benzoate": "93-89-0",
    "benzyl cinnamate": "103-41-3",
    "cinnamyl butyrate": "103-61-7",
    "phenyl propyl aldehyde": "104-53-0",
    "hydroquinone dimethyl ether": "150-78-7",
    "indole": "120-72-9",
    "phenyl cresyl oxide": "139-02-6",
    "phenyl ethyl phenylacetate": "2114-33-2",
    "citronellyl phenylacetate": "103-48-0",
    "eugenyl phenylacetate": "7783-13-1",
    "methyl phenylacetate": "101-41-7",
    "rosemary": "8000-25-7",
    "thyme": "8007-46-3",
    "nutmeg": "8008-45-5",
    "mace": "8007-40-1",
    "lemongrass": "8007-02-1",
    "peppermint": "8006-90-4",
    "eucalyptus": "8000-48-4",
    "caraway": "8000-42-8",
    "fennel": "8006-84-6",
    "coriander": "8008-52-4",
    "galbanum resin": "8023-91-4",
    "galbanum": "8023-91-4",
    "galbanum oil": "8023-91-4",
    "citronellyl formate": "105-85-1",
    "decyl formate": "5451-52-5",
    "ethyl acetoacetate": "141-97-9",
    "ethyl acetate": "141-78-6",
    "octyl acetate": "112-14-1",
    "terpinyl acetate": "8007-35-0",
    "neryl acetate": "141-12-8",
    "geranyl propionate": "105-90-8",
    "linalyl butyrate": "78-36-4",
    "citronellyl butyrate": "141-16-2",
    "geranyl butyrate": "106-29-6",
    "phenyl ethyl butyrate": "103-52-6",
    "citronellyl propionate": "141-14-0",
    "farnesol": "4602-84-0",
    "nerolidol": "7212-44-4",
    "bisabolol": "23089-26-1",
    "damascone alpha": "43052-91-7",
    "damascenone": "23696-85-7",
    "ionone methyl": "1335-46-2",
    "methyl dihydrojasmonate": "24851-98-7",
    "hedione": "24851-98-7",
    "lilial": "80-54-6",
    "lyral": "31906-04-4",
    "hydroxyisohexyl 3-cyclohexene carboxaldehyde": "31906-04-4",
    "galaxolide": "1222-05-5",
    "fixolide": "21145-77-7",
    "musk t": "105-95-3",
    "ambrettolide": "123-69-3",
    "ethylene brassylate": "105-95-3",
    "exaltolide": "502-72-7",
    "pentadecalactone": "502-72-7",
    "lactiscene": "28645-51-4",
    "iso e super": "68555-14-8",
    "vertofix": "32388-55-9",
    "methyl cedryl ketone": "32388-55-9",
    "isobornyl acetate": "125-12-2",
    "vetiveryl acetate": "62563-80-8",
    "cinnamyl acetate": "103-54-8",
    "methyl octine carbonate": "111-80-8",
    "santalyl phenylacetate": "1323-75-7",
}


def match_name_to_cas(name: str) -> str | None:
    """Match a Poucher ingredient name to CAS."""
    norm = name.lower().strip()
    norm = re.sub(r'[^\w\s]', '', norm)
    norm = re.sub(r'\s+', ' ', norm)

    if norm in POUCHER_NAME_TO_CAS:
        return POUCHER_NAME_TO_CAS[norm]

    # Try conservative qualifier stripping before broader prefix matching.
    # This catches origin/style suffixes such as "rosemary french" without
    # letting "rose" match "rosemary" or "pepper" match "peppermint".
    suffix_tokens = {
        "african", "american", "bigarade", "bourbon", "bulgarian", "ceylon",
        "distilled", "french", "italian", "java", "japanese", "manilla",
        "para", "red", "white",
    }
    parts = norm.split()
    while len(parts) > 1 and parts[-1] in suffix_tokens:
        parts = parts[:-1]
        shortened = " ".join(parts)
        if shortened in POUCHER_NAME_TO_CAS:
            return POUCHER_NAME_TO_CAS[shortened]

    # Try phrase-prefix matches only on token boundaries, longest first.
    for hint, cas in sorted(POUCHER_NAME_TO_CAS.items(), key=lambda item: len(item[0]), reverse=True):
        if norm.startswith(hint + " "):
            return cas

    return None


def main():
    pages = load_pages()
    print(f"Loaded {len(pages)} Poucher pages")

    # Extract all odour classification entries
    all_entries = []
    seen = set()  # (name_lower, tier) to avoid duplicates

    for page in pages:
        entries = parse_odour_classification(page["text"])
        for entry in entries:
            key = (entry["name"].lower(), entry["tier"])
            if key not in seen:
                seen.add(key)
                all_entries.append(entry)

    print(f"Extracted {len(all_entries)} unique material-tier entries")

    # Count by tier
    tier_counts = defaultdict(int)
    for e in all_entries:
        tier_counts[e["tier"]] += 1
    for tier in ['top', 'mid', 'base']:
        print(f"  {tier:5s}: {tier_counts[tier]}")

    # Match to CAS
    cas_to_tiers = defaultdict(set)
    matched = 0
    unmatched = []

    for entry in all_entries:
        cas = match_name_to_cas(entry["name"])
        if cas:
            cas_to_tiers[cas].add(entry["tier"])
            matched += 1
        else:
            unmatched.append(entry["name"])

    # Resolve multi-tier (prefer most volatile)
    cas_to_tier_final = {}
    for cas, tiers in cas_to_tiers.items():
        if 'top' in tiers:
            cas_to_tier_final[cas] = 'top'
        elif 'mid' in tiers:
            cas_to_tier_final[cas] = 'mid'
        else:
            cas_to_tier_final[cas] = 'base'

    print(f"\nCAS matching:")
    print(f"  Matched: {matched} entries → {len(cas_to_tier_final)} unique CAS")
    tier_dist = defaultdict(int)
    for tier in cas_to_tier_final.values():
        tier_dist[tier] += 1
    for tier in ['top', 'mid', 'base']:
        print(f"  {tier:5s}: {tier_dist[tier]}")
    print(f"  Unmatched material names: {len(unmatched)}")
    if unmatched:
        unique_unmatched = sorted(set(unmatched))[:20]
        print(f"    Samples: {unique_unmatched}")

    # Save
    output = {
        "source": "Poucher Vol II - Odour Classification tables",
        "total_entries": len(all_entries),
        "unique_cas": len(cas_to_tier_final),
        "cas_to_tier": {k: v for k, v in sorted(cas_to_tier_final.items())},
        "tier_distribution": dict(tier_dist),
        "unmatched_names": sorted(set(unmatched)),
    }
    (DATA / "poucher_tier_lookup_expanded.json").write_text(json.dumps(output, indent=2))
    print(f"\nSaved to data/poucher_tier_lookup_expanded.json")

    # Merge with existing perfumer tier lookup
    with open(DATA / "perfumer_tier_lookup.json") as f:
        existing = json.load(f)

    merged = dict(existing)
    for cas, tier in cas_to_tier_final.items():
        if cas not in merged:
            merged[cas] = tier

    (DATA / "perfumer_tier_lookup.json").write_text(
        json.dumps({k: v for k, v in sorted(merged.items())}, indent=2)
    )
    print(f"Merged lookup: {len(existing)} → {len(merged)} CAS")
    tier_dist_merged = defaultdict(int)
    for tier in merged.values():
        tier_dist_merged[tier] += 1
    for tier in ['top', 'mid', 'base']:
        print(f"  {tier:5s}: {tier_dist_merged[tier]}")


if __name__ == "__main__":
    main()