#!/usr/bin/env python3 """Convert sanitized WiseMoor formulas into PINO training candidates. This is intentionally gated: sanitized WiseMoor rows are name/amount formulas, while the core PINO training set needs CAS-resolved, normalized formulas. The script writes only formulas that meet configurable resolution thresholds and also writes an audit file for the unresolved materials. """ from __future__ import annotations import argparse import json import re import sys import unicodedata from collections import Counter, defaultdict from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from ingest_tgsc import build_tgsc_mapping, map_ingredient # noqa: E402 from pino.registry import AromaRegistry # noqa: E402 WISEMOOR_NAME_OVERRIDES: dict[str, str] = { # Solvents / carriers. "dep": "84-66-2", # diethyl phthalate "d.p.g": "25265-71-8", "dpg": "25265-71-8", "dipropylene glycol": "25265-71-8", "t.e.c": "77-93-0", # triethyl citrate "i.p.m": "110-27-0", # isopropyl myristate # WiseMoor spelling/order variants for common aroma chemicals. "acetate benzyl": "140-11-4", "acetate bornyl": "76-49-3", "acetate bornyl l": "76-49-3", "acetate butyl": "123-86-4", "acetate citronellyl": "150-84-5", "acetate d.m.b.c": "151-05-3", "dmbc acetate": "151-05-3", "acetate ethyl": "141-78-6", "acetate geranyl": "105-87-3", "acetate hexenyl cis-3": "3681-71-8", "acetate hexyl": "142-92-7", "acetate isoamyl": "123-92-2", "acetate amyl": "628-63-7", "amyl acetate": "628-63-7", "acetate cinnamic": "103-54-8", "acetate linalyl": "115-95-7", "acetate linallyl": "115-95-7", "acetate neryl": "141-12-8", "acetate octyl": "112-14-1", "acetate prenyl": "1191-16-8", "acetate styrallyl": "93-92-5", "acetate styralyl": "93-92-5", "acetate terpenyl": "80-26-2", "acetate terpene": "80-26-2", "acetophenone": "98-86-2", "aldehyde alpha hexyl cinnamic": "101-86-0", "aldehyde alpha amyl cinnamic": "122-40-7", "aldehyde anisic": "123-11-5", "aldehyde c 08": "124-13-0", "aldehyde c 09": "112-31-2", "aldehyde c 11 und": "112-45-8", "aldehyde c 18": "104-61-0", "aldehydec 14": "104-67-6", "aldehyde cinammic": "104-55-2", "aldehyde cinnamic": "104-55-2", "aldehyde cyclamen": "103-95-7", "alcohol c-06": "111-27-3", "alcohol cinnamic": "104-54-1", "alcool cinnamique": "104-54-1", "alcohol isoamyl": "123-51-3", "alcohol phenylethyl": "60-12-8", "alcool phenylpropylique": "122-97-4", "alyl amyl glycolate": "67634-20-2", "allyl amyl glycolate": "67634-20-2", "anethol": "104-46-1", "anisic aldehyde": "123-11-5", "anthranilate dimethyl": "85-74-5", "anthranilate methyl": "134-20-3", "benzyl propionate": "122-63-4", "benzyl cinnamate": "103-41-3", "benzylic alcohol": "100-51-6", "benzophenone": "119-61-9", "benzoat benzyl": "120-51-4", "benzoate benzyl": "120-51-4", "benzoate methyle": "93-58-3", "bht": "128-37-0", "butyl quinoleine sec": "65442-31-1", "c 7 aldehyde": "111-71-7", "c 8 aldehyde": "124-13-0", "c 10 aldehyde": "112-31-2", "c 11 aldehyde": "112-44-7", "c 12 l aldehyde": "112-54-9", "c 12 mna": "110-41-8", "c 12 mna aldehyde": "110-41-8", "c 14 aldehyde": "104-67-6", "c 16 aldehyde": "77-83-8", "c 18 aldehyde": "104-61-0", "calone": "28940-11-6", "calone 1951": "28940-11-6", "cedramber": "67874-81-1", "cedrenol": "28231-03-0", "cedryl acetate": "77-54-3", "cimene para": "99-87-6", "citronellal": "106-23-0", "citronnellal": "106-23-0", "citronellol": "106-22-9", "citronnellol": "106-22-9", "citronelol": "106-22-9", "citronellool": "106-22-9", "citronellyl acetate": "150-84-5", "citronnellyl acetate": "150-84-5", "citronellyl formate": "105-85-1", "citronnellyl formate": "105-85-1", "cis 3 hexenol": "928-96-1", "cis-3 hexenol": "928-96-1", "hexenol cis-3": "928-96-1", "cis 3 hexenyl acetate": "3681-71-8", "cis 3 hexenyl benzoate": "25152-85-6", "cis 3 hexenyl salicylate": "65405-77-8", "cis-3 hexenyl salicylate": "65405-77-8", "cis jasmone": "488-10-8", "clonal": "106-72-9", "damascenone": "23696-85-7", "damascenone total": "23696-85-7", "damascone alpha": "43052-87-5", "alpha damascone": "43052-87-5", "damascone beta": "35044-57-6", "beta damascone": "35044-57-6", "damascone delta": "57378-72-0", "decalactone gamma": "706-14-9", "gamma decalactone": "706-14-9", "dihidromyrcenol": "18479-58-8", "dimethyl anthranylate": "85-74-5", "eucalyptol": "470-82-6", "eucaliptol": "470-82-6", "exaltex": "106-02-5", "estragol": "140-67-0", "estragone": "140-67-0", "floralozone": "67634-15-5", "frambinone": "5471-51-2", "heptalactone gamma": "105-21-5", "heptoate allyl": "142-19-8", "helvetolide": "141773-73-1", "hivernal neo": "300371-33-9", "hydoxycitronnellal": "107-75-5", "hidroxycitronellal": "107-75-5", "hidroxicitronelal": "107-75-5", "hydroxycitronnellal": "107-75-5", "indol": "120-72-9", "iso & super": "54464-57-2", "isobutyl quinoleine": "65442-31-1", "isobutylquinolein": "65442-31-1", "isobutylquinoleine": "65442-31-1", "isobornylcyclohexanol": "3407-42-9", "isobornyl cyclohexanol": "3407-42-9", "jasmonal h": "101-86-0", "jasmonal": "101365-33-7", "jessemal": "38285-49-3", "limonene": "5989-27-5", "macrolide": "106-02-5", "methyl anthranylate": "134-20-3", "methyl heptenone": "110-93-0", "methyl octin carbonate": "111-80-8", "methyl octine carbonate": "111-80-8", "methylnaphtylcetone": "93-08-3", "methyl naphtyl cetone": "93-08-3", "musc ambrette substitute": "83-66-9", "mus ambrette substitute": "83-66-9", "musc ketone": "81-14-1", "musk t": "105-95-3", "musc t": "105-95-3", "musc xylol": "81-15-2", "neo heliopan av": "5466-77-3", "nopyl acetate": "128-51-8", "norlimbanol": "70788-30-6", "oxyphenylon": "5471-51-2", "oxide rose l": "16409-43-1", "phenylacetic aldehyde": "122-78-1", "phenylpropyl alcohol": "122-97-4", "phenylpropylic alcohol": "122-97-4", "pinene alpha": "80-56-8", "pinene beta": "127-91-3", "rosalva": "13019-22-2", "rosacetol": "90-17-5", "rose oxyde": "16409-43-1", "rose oxyde d": "16409-43-1", "rose oxyde l": "16409-43-1", "salicilate hexenyl cis-3": "65405-77-8", "salicilate octyl": "118-60-5", "salicylate octyl": "118-60-5", "sandalor": "65113-99-5", "sandela": "3407-42-9", "sandranol": "28219-60-5", "terpinene gamma": "99-85-4", "terpenyl acetate": "80-26-2", "trimofix o": "144020-22-4", "veltol": "118-71-8", "veltol plus": "4940-11-8", "vertenex": "32210-23-4", "vetyver acetate": "117-98-6", "vetyveryle acetate": "117-98-6", "vetyveryl acetate": "117-98-6", "adoxal": "141-13-9", "amarocit": "67674-46-8", "amber extreme": "476332-65-7", "amberketal": "57345-19-4", "ambrinol": "41199-19-3", "andrane": "13567-39-0", "aurantiol": "89-43-0", "bourgeonal": "18127-01-0", "bourgeonnal": "18127-01-0", "brahmanol": "72089-08-8", "celeriax": "28231-03-0", "costus oliffac": "68917-29-3", "cuminic aldehyde": "122-03-2", "dimetol": "13254-34-7", "dynascone": "56973-85-4", "ethyl amyl ketone": "541-85-5", "hexalon": "79-78-7", "jacinth body": "1205-17-0", "koavone": "81786-73-4", "mandarinal": "540-18-1", "methylparacresol": "106-44-5", "methyl paracresol": "106-44-5", "methylpamplemousse": "67674-46-8", "mousse metra": "4707-47-5", "myrac aldehyde": "37677-14-8", "osyrane": "41890-92-0", "osyrol": "41890-92-0", "paracresol": "106-44-5", "paracresyl acetate": "140-39-6", "paracresyl phenylacetate": "101-94-0", "peryl aldehyde": "2111-75-3", "phenylethyl phenylacetate": "102-20-5", "phenylethyl dimethyl carbinol": "103-05-9", "prunolide": "104-61-0", "rhubafuran": "82461-14-1", "scatol": "83-34-1", "stemone": "22457-23-4", "syringa aldehyde": "104-87-0", "timberol": "70788-30-6", "vetylione": "79-89-0", "viridine": "101-48-4", # Existing natural-complex profiles. "ambreine pure": "NATURAL:68916-26-7", "animalys": "NATURAL:68916-26-7", "civet cnc": "NATURAL:68916-26-7", "civette synthetic": "NATURAL:68916-26-7", "civette synthetic / ambreine": "NATURAL:68916-26-7", "ambrarome": "NATURAL:8023-83-4", "ambrarome absolute": "NATURAL:8023-83-4", "ambrarome resinoid": "NATURAL:8023-83-4", "cardamome oil": "NATURAL:8000-66-6", "cardamome": "NATURAL:8000-66-6", "caraway": "99-49-0", "caryophyllene rect": "87-44-5", "cariophyllene rect": "87-44-5", "caryphyllene rect": "87-44-5", "cariophylene rect": "87-44-5", "cinnamon leaves": "97-53-0", "cinnamon leaves oil": "97-53-0", "cinnamon ceylon": "97-53-0", "cloves oil": "97-53-0", "clove bud": "97-53-0", "cumin oil": "122-03-2", "cumin eo": "122-03-2", "cumin seeds": "122-03-2", "cedarwood": "77-53-2", "cedarwood oil": "77-53-2", "cedarwood virginia": "77-53-2", "cedar virginia": "77-53-2", "clary sage oil": "115-95-7", "clary sage absolute oil": "115-95-7", "officinalis sage oil": "115-95-7", "armoise": "8023-88-9", "artemisia": "8023-88-9", "artemisia oil": "8023-88-9", "basilic": "8015-73-4", "basilic oil": "8015-73-4", "bigarade oil": "NATURAL:8014-17-3", "ciste oil": "8016-26-0", "ciste absolute": "8016-26-0", "citronella ceylon": "8000-29-1", "citronnella oil": "8000-29-1", "citronela ceylan": "8000-29-1", "copahu balsam": "8001-61-4", "coriander": "NATURAL:8008-52-4", "coriander oil": "NATURAL:8008-52-4", "gaiacwood": "NATURAL:8016-23-7", "galbanum synthetic r": "68912-13-0", "ginger co2": "8007-08-7", "guaiacwood": "NATURAL:8016-23-7", "gurjum balsam": "8030-55-5", "hysope oil": "84603-66-7", "immortelle absolute": "90131-24-1", "iris concrete": "90045-43-5", "irone": "79-69-6", "irone alpha": "79-69-6", "jasmin": "38285-49-3", "jasmin base": "38285-49-3", "laurus nobilis leaf": "8006-78-8", "liatrix absolute": "68606-83-7", "hyperessence liatrix": "68606-83-7", "lavander": "NATURAL:8000-28-0", "lavander oil": "NATURAL:8000-28-0", "lavander oil 40/42": "NATURAL:8000-28-0", "lavandin abryal": "NATURAL:8000-28-0", "lavandin abryal oil": "NATURAL:8000-28-0", "lavandin super oil": "NATURAL:8000-28-0", "mandarin": "NATURAL:8008-31-9", "mandarine": "NATURAL:8008-31-9", "mandarine oil": "NATURAL:8008-31-9", "mandarin vert": "NATURAL:8008-57-9", "mandarin verte": "NATURAL:8008-57-9", "mandarin italiana": "NATURAL:8008-57-9", "mandarine verte italie he": "NATURAL:8008-57-9", "mirceno": "123-35-3", "myrceno": "123-35-3", "myrte oil": "NATURAL:8000-25-7", "oakmoss": "90028-68-5", "oakmoss absolute": "90028-68-5", "absolute oakmoss": "90028-68-5", "orange brasil": "NATURAL:8008-57-9", "orange seet brasil oil": "NATURAL:8008-57-9", "orange sweet brasil oil": "NATURAL:8008-57-9", "patchouli decol": "5986-55-0", "patchouly": "5986-55-0", "patchouly dm": "5986-55-0", "patchouly dm oil": "5986-55-0", "patchouly oil": "5986-55-0", "petit grain": "NATURAL:8014-17-3", "petit grain oil": "NATURAL:8014-17-3", "petit grain paraguay": "NATURAL:8014-17-3", "pink pepper": "NATURAL:8000-25-7", "pepper oil": "NATURAL:8000-25-7", "pimento oil": "8006-39-1", "rose": "NATURAL:8007-01-0", "sandalwood oil": "NATURAL:8006-87-9", "vetyver oil": "NATURAL:8016-96-4", "vetiver haiti": "NATURAL:8016-96-4", "white thyme oil": "97-53-0", "white thyme": "97-53-0", "red thyme": "97-53-0", "red thyme oil": "97-53-0", "spearmint": "8008-79-5", "spearmint oil": "8008-79-5", "styrax oil": "8046-42-0", "taget oil": "8016-84-0", "tagets": "8016-84-0", "tarragon oil": "8016-88-4", "tonka beans absolute": "8024-04-2", "tonka beans resinoi": "8024-04-2", "violet leaves absolute": "8024-08-6", "ylang extra": "NATURAL:8006-81-3", "ylang extra oil": "NATURAL:8006-81-3", "ylang oil": "NATURAL:8006-81-3", "ylang substitute": "NATURAL:8006-81-3", "ylang ylang extra": "NATURAL:8006-81-3", "ylang ylang substitute": "NATURAL:8006-81-3", # Long-tail WiseMoor OCR/French/trade-name variants needed for row coverage. "citrone lool": "106-22-9", "citronelool": "106-22-9", "fleur d oranger base": "NATURAL:8014-17-3", "fixateur": "110-27-0", "fixateur 404": "110-27-0", "bornyl acetate": "76-49-3", "aspic": "NATURAL:8000-28-0", "aspic oil from spain": "NATURAL:8000-28-0", "blue camomile": "8015-92-7", "camomile blue": "8015-92-7", "roman camomile": "8015-92-7", "roman camomile oil": "8015-92-7", "camomile oil roman": "8015-92-7", "camomille romaine": "8015-92-7", "rose acetate": "141-12-8", "acetanisol crist": "100-06-1", "azarbre": "54464-57-2", "globalide": "34902-57-3", "magnolan": "27606-09-3", "salicylate benzyl": "118-58-1", "anise star": "8007-70-3", "formiate citronellyl": "105-85-1", "formiate de citronnellyle": "105-85-1", "formiate geranyl": "105-86-2", "nutmeg eo": "8008-45-5", "peppermint usp": "8006-90-4", "peppermint usa": "8006-90-4", "patchoily oil": "5986-55-0", "patchouli": "5986-55-0", "patchouly indo": "5986-55-0", "isobornyl acetate": "125-12-2", "ambrocenide": "211299-54-6", "ambrocenide 10": "211299-54-6", "cyclohexyl propionate allyl": "2705-87-5", "ciclohexyl propionate allyl": "2705-87-5", "allyl ciclohexyl propionate": "2705-87-5", "farenal": "141-13-9", "orange terpenes": "5989-27-5", "salicilate benzyl": "118-58-1", "salicilate hexyl": "6259-76-3", "salicilate amyl": "2050-08-0", "salicilate isoamyl": "87-20-7", "ally amyl glycolate": "67634-20-2", "allyl amyl glicolate": "67634-20-2", "aldehyde mandarin": "540-18-1", "lavandin grosso": "NATURAL:8000-28-0", "lavandin grosso oil": "NATURAL:8000-28-0", "lavandin": "NATURAL:8000-28-0", "lavandin abrialys oil": "NATURAL:8000-28-0", "acetate geranilo": "105-87-3", "acetyl methyl carbinol": "513-86-0", "butirate ethyl": "105-54-4", "caprilate ethyl": "106-32-1", "methyl butirate ethyl": "7452-79-1", "safranate ethyl": "35044-59-8", "2 methyl butyl acetate": "624-41-9", "elemi": "8023-89-0", "salvia sclarea french": "8016-63-5", "grapefruit": "NATURAL:8008-57-9", "acetate citronelyl": "150-84-5", "oxide caryophylene": "1139-30-6", "acid acetic": "64-19-7", "methyll heptenone": "110-93-0", "mircene": "123-35-3", "pheylethyl alcohol": "60-12-8", "phenylrthyl alcohol": "60-12-8", "phenylethyl alcohool": "60-12-8", "alcool phenylethylique": "60-12-8", "alcohol 39c": "60-12-8", "butirate d m b c": "10094-34-5", "dmbc butyrate": "10094-34-5", "fructalate": "72903-27-6", "monopropilenglicol": "57-55-6", "neobutenone alpha": "56973-85-4", "oriclone": "16587-71-6", "salicilate octilo": "118-60-5", "zenolide": "54982-83-1", "eucalyptus globulus 80": "470-82-6", "eucalyptus globulus oil": "470-82-6", "eucalyptus": "470-82-6", "diphenyl oxide": "101-84-8", "diphenyl oxyde": "101-84-8", "aldehyde c8": "124-13-0", "methyl dihydro jasmonate": "24851-98-7", "helvetolide pur": "141773-73-1", "canthoxal": "5462-06-6", "canthoxal iff": "5462-06-6", "cis 2 hxenyl acetate": "2497-18-9", "ethyl isovalerate": "108-64-5", "orange brazil": "NATURAL:8008-57-9", "ethyl aceto acetate": "141-97-9", "hidroxycitrone lool": "106-22-9", "hidroxycitronelool": "106-22-9", "hydroxycitonnellal": "107-75-5", "floropal": "5182-36-5", "d h m": "18479-58-8", "dihydrmyrcenol": "18479-58-8", "menta arvensis": "8006-90-4", "ciste labdanum sis absolute": "8016-26-0", "abs ciste labdanum": "8016-26-0", "cinnamyle acetate": "21040-45-9", "paramethyquinoleine": "491-35-0", "tetrahydroparamethylquinoleine": "635-46-1", "caprylic acid": "124-07-2", "linalyl propinate": "144-39-8", "birch tree oil": "8001-88-5", "valerianic acid": "109-52-4", "methylhexylcetone": "111-13-7", "carvyl acetate": "97-42-7", "phenylrthyl acetate": "103-45-7", "methyl phenyl acetate": "101-41-7", "methyl phenylacetate": "101-41-7", "phenylethyl isobutyrate": "103-48-0", "methyl phenylacetate": "101-41-7", "menthone": "10458-14-7", "isomenthone": "491-07-6", "rosalva iff": "13019-22-2", "ethyl citrate": "77-93-0", "benzyl butyrate": "103-37-7", "heliotrpin": "120-57-0", "heliotropin": "120-57-0", "gamma octalactone": "104-50-7", "c 12 l aldehyde 50": "112-54-9", # Formula-by-formula recovery for WiseMoor rows rejected by the coverage gate. "acetate de linalyle": "115-95-7", "acetate linalyle": "115-95-7", "acetate d isobornyle": "125-12-2", "ylang s": "NATURAL:8006-81-3", "geraniul egypt oil": "NATURAL:8007-01-0", "coriandre russe": "NATURAL:8008-52-4", "cypres biolande": "NATURAL:8014-17-3", "salicylate d amyle": "2050-08-0", "romarin": "NATURAL:8000-25-7", "thym rouge": "97-53-0", "cumin": "122-03-2", "black pepper": "87-44-5", "black pepper oil": "87-44-5", "cypress oil": "NATURAL:8014-17-3", "vetiver oil haiti": "NATURAL:8016-96-4", "vetiver oil bourbon": "NATURAL:8016-96-4", "cistus oil spain": "8016-26-0", "vetival": "117-98-6", "santalex": "NATURAL:8006-87-9", "tetrahtdromugol": "78-69-3", "tetrahydromugol": "78-69-3", "methyl pamplemousse giv": "67674-46-8", "veloutone": "65443-14-3", "veloutone firm": "65443-14-3", "bourgennal": "18127-01-0", "benzyl salicilate": "118-58-1", "magnolia oil lmr": "24851-98-7", "muscone": "541-91-3", "neo heliopan bb": "131-57-7", "abs jasmin maroc": "38285-49-3", "viridine padma": "101-48-4", "ethyl acetyl acetate": "141-97-9", "jasmin 1103": "38285-49-3", "dimethyl heptanol": "106-21-8", "myrthe oil": "NATURAL:8000-25-7", "lime distilled": "NATURAL:8008-57-9", "phenylethy alcohol": "60-12-8", "tagete oil": "8016-84-0", "marjolaine": "NATURAL:8000-25-7", "thyme oil": "97-53-0", "marjoliane dl": "NATURAL:8000-25-7", "orange": "NATURAL:8008-57-9", "vetyverol": "117-98-6", "verbena oil": "106-22-9", "clary sage absolute": "8016-63-5", "cedar leaf oil": "8007-20-3", "ylang absolute oil": "NATURAL:8006-81-3", "civet synth firm": "NATURAL:68916-26-7", "jasmin 231 type": "38285-49-3", "musc k": "81-14-1", "oakmoss absolute solute": "90028-68-5", "rose absolute type": "NATURAL:8007-01-0", "ambretolide": "7779-50-2", "cedarwood texas oil": "NATURAL:8006-87-9", "cedarwood oil texas": "NATURAL:8006-87-9", "dimethyl phenyl ethyl carbynol": "103-05-9", "dipropylen glycol": "25265-71-8", "isopropyl miristate": "110-27-0", "iso propyl myristate": "110-27-0", "sage sclarea oil": "8016-63-5", "vetyvert haiti oil": "NATURAL:8016-96-4", "ambreine": "NATURAL:68916-26-7", "camphre": "76-22-2", "cedarleaf oil": "8007-20-3", "copaiba balsam oil": "8001-61-4", "cuminaldehyde": "122-03-2", "labdanum oil": "8016-26-0", "tonka bean absolute": "8024-04-2", "cedre he bois atlas": "NATURAL:8006-87-9", "santal he inde": "NATURAL:8006-87-9", "acetate cedryle": "77-54-3", "vetyver ess java": "NATURAL:8016-96-4", "petitgrain he paraguay": "NATURAL:8014-17-3", "salicylate cis 3 hexenyle": "65405-77-8", "phenylethyl formate": "104-62-1", "amyl butyrate iso": "106-27-4", "ethyl propionate": "105-37-3", "boccoxim": "81782-77-6", "methyloctalactone": "39212-23-2", "clary sage absolute oil": "8016-63-5", "c 19 aldehyde": "104-67-6", "speatmint": "8008-79-5", "dihydrojasmone": "95-41-0", "taget absolute": "8016-84-0", "irisone alpha giv": "79-69-6", "rose bulgarian oil reconstitued": "NATURAL:8007-01-0", "ylang extra absolute": "NATURAL:8006-81-3", "carbitol": "111-90-0", "methyl cyclo pentenolene butyrate": "80-71-7", "sauge sclaree russe": "8016-63-5", "girofles clous": "97-53-0", "acetate de neryle": "141-12-8", "acetate de geranyle": "105-87-3", "acetate de citronnellyle": "150-84-5", "gurjum oil": "8030-55-5", "poivre": "87-44-5", "oxyde de rose d": "16409-43-1", "isobutyl isobutyrate": "97-85-8", "citronnelle java": "8000-29-1", "orange florida": "NATURAL:8008-57-9", "gamma terpinene": "99-85-4", "dilmrthyl anthranylate": "85-74-5", "thymol": "89-83-8", "alcool phenylethylique": "60-12-8", "geraniol": "106-24-1", "menthyl acetate": "89-48-5", "menthe bresil": "8006-90-4", "c 10 adehyde": "112-31-2", "citeronnellyl acetate": "150-84-5", "ethyl caproate": "123-66-0", "aldehyde c 7 e": "111-71-7", "methyl nonyl ketone": "112-12-9", "isopulegol purified": "89-79-2", "para cymene": "99-87-6", "ald c 11 undecylic": "112-44-7", "tangerine oil": "NATURAL:8008-57-9", "dioctyl adipate": "123-79-5", "dipentene": "5989-27-5", "lime terpenes": "5989-27-5", "c 9 aldehyde": "112-31-2", "vetyver terpenes": "NATURAL:8016-96-4", "animalys 1745": "NATURAL:68916-26-7", "butyric acid": "107-92-6", "amyris oil": "NATURAL:8006-87-9", "bouleau oil rectified": "8001-88-5", "beta pinene": "127-91-3", "gappefruit body": "NATURAL:8008-57-9", "amboxan henkel": "3738-00-9", "dynamone cas 8016 26 0": "8016-26-0", "dimyrcetol": "18479-58-8", "iso amyl butyrate": "106-27-4", "paratoluic aldehyde": "104-87-0", "hexyl formate": "629-33-4", "hexyl butyrate": "2639-63-6", "iso pulegol": "89-79-2", "citronnellyl isobutyrate": "97-89-2", "estragon oil": "8016-88-4", "diphenylmethane": "101-81-5", "c 9 alcohol": "143-08-8", "iso valerianic acid": "503-74-2", "hydratropic aldehyde dimethylacetal": "90-87-9", "hey absolute": "91-64-5", "galbex": "68912-13-0", "rosone bba": "16409-43-1", "cortex aldehyde": "68039-49-6", "c 11 alcool lenique": "112-42-5", "citeronnellyl acetate": "150-84-5", "methyl 2 pyrazinz": "109-08-0", "dibutyl sulfure iff": "544-40-1", } def _wisemoor_lookup_key(name: str) -> str: norm = unicodedata.normalize("NFKD", str(name).strip().lower()) norm = "".join(ch for ch in norm if not unicodedata.combining(ch)) norm = re.sub(r"[^a-z0-9&]+", " ", norm) return " ".join(norm.split()) WISEMOOR_NORMALIZED_OVERRIDES = { _wisemoor_lookup_key(raw): cas for raw, cas in WISEMOOR_NAME_OVERRIDES.items() } def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--input", type=Path, default=Path("data/wisemoor_free_formulas.sanitized.jsonl")) parser.add_argument("--output", type=Path, default=Path("data/wisemoor_training_candidates.jsonl")) parser.add_argument("--audit-output", type=Path, default=Path("artifacts/wisemoor_training_candidate_audit.json")) parser.add_argument("--min-row-coverage", type=float, default=0.80) parser.add_argument("--min-amount-coverage", type=float, default=0.80) parser.add_argument("--min-resolved-components", type=int, default=3) return parser.parse_args() def load_jsonl(path: Path) -> list[dict[str, Any]]: with path.open(encoding="utf-8") as handle: return [json.loads(line) for line in handle if line.strip()] def normalize_formula(parts_by_cas: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: total = sum(float(item["amount"]) for item in parts_by_cas.values()) if total <= 0: return [] formula = [] for cas, item in sorted(parts_by_cas.items()): formula.append( { "cas": cas, "name": item["name"], "weight_fraction": float(item["amount"]) / total, } ) return formula def resolve_wisemoor_name(name: str, mapping: dict[str, str]) -> str | None: norm = " ".join(str(name).strip().lower().split()) if norm in WISEMOOR_NAME_OVERRIDES: return WISEMOOR_NAME_OVERRIDES[norm] lookup = _wisemoor_lookup_key(name) if lookup in WISEMOOR_NORMALIZED_OVERRIDES: return WISEMOOR_NORMALIZED_OVERRIDES[lookup] compact = lookup.replace(" ", "") if compact in WISEMOOR_NORMALIZED_OVERRIDES: return WISEMOOR_NORMALIZED_OVERRIDES[compact] cas = map_ingredient(name, mapping) if cas: return cas aldehyde_match = re.match(r"^c\s+(\d+)(?:\s+([a-z]+))?\s+aldehyde$", norm) if aldehyde_match: chain, qualifier = aldehyde_match.groups() candidates = [ f"aldehyde c-{chain} {qualifier}".strip(), f"aldehyde c-{chain}", ] for candidate in candidates: if candidate in WISEMOOR_NAME_OVERRIDES: return WISEMOOR_NAME_OVERRIDES[candidate] cas = map_ingredient(candidate, mapping) if cas: return cas return None def convert_record( record: dict[str, Any], mapping: dict[str, str], *, min_row_coverage: float, min_amount_coverage: float, min_resolved_components: int, ) -> tuple[dict[str, Any] | None, dict[str, Any]]: raw_rows = record.get("formula", {}).get("formula_rows") or [] rows = [ row for row in raw_rows if re.search(r"[A-Za-z]", str(row.get("ingredient") or "")) and float(row.get("amount") or 0.0) > 0 ] total_rows = len(rows) total_amount = sum(float(row.get("amount") or 0.0) for row in rows) parts_by_cas: dict[str, dict[str, Any]] = {} unresolved: list[dict[str, Any]] = [] resolved_rows = 0 resolved_amount = 0.0 for row in rows: name = str(row.get("ingredient") or "").strip() amount = float(row.get("amount") or 0.0) if not name or amount <= 0: continue cas = row.get("cas") or resolve_wisemoor_name(name, mapping) if cas: resolved_rows += 1 resolved_amount += amount existing = parts_by_cas.setdefault(cas, {"name": name, "amount": 0.0}) existing["amount"] += amount else: unresolved.append({"ingredient": name, "amount": amount}) row_coverage = resolved_rows / total_rows if total_rows else 0.0 amount_coverage = resolved_amount / total_amount if total_amount else 0.0 formula = normalize_formula(parts_by_cas) accepted = ( row_coverage >= min_row_coverage and amount_coverage >= min_amount_coverage and len(formula) >= min_resolved_components ) audit = { "product_id": record.get("product_id"), "name": record.get("name"), "total_rows": total_rows, "resolved_rows": resolved_rows, "row_coverage": row_coverage, "total_amount": total_amount, "resolved_amount": resolved_amount, "amount_coverage": amount_coverage, "resolved_components": len(formula), "accepted": accepted, "unresolved": unresolved, } if not accepted: return None, audit candidate = { "formula_id": f"WISEMOOR_{record.get('product_id')}", "source": "wisemoor_labtorium_free_sanitized", "source_url": record.get("source_url"), "name": record.get("name"), # Preserve source-observed formula text as provenance. This is useful # for retrieval and later human/AI-assisted extraction, but is not an # assessor label and must not be promoted to one implicitly. "description": record.get("description"), "genre": "wisemoor_formula", "notes": record.get("notes"), "formula": formula, "metadata": { "product_id": record.get("product_id"), "slug": record.get("slug"), "row_coverage": row_coverage, "amount_coverage": amount_coverage, "unresolved_count": len(unresolved), "sanitized_source": str(Path("data/wisemoor_free_formulas.sanitized.jsonl")), "description_provenance": ( "source_observed_wisemoor_product_text" if record.get("description") else None ), }, } return candidate, audit def write_jsonl(path: Path, records: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: for record in records: handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") def main() -> None: args = parse_args() registry = AromaRegistry() mapping = build_tgsc_mapping(registry) candidates: list[dict[str, Any]] = [] audits: list[dict[str, Any]] = [] unresolved_counter: Counter[str] = Counter() for record in load_jsonl(args.input): candidate, audit = convert_record( record, mapping, min_row_coverage=args.min_row_coverage, min_amount_coverage=args.min_amount_coverage, min_resolved_components=args.min_resolved_components, ) audits.append(audit) for item in audit["unresolved"]: unresolved_counter[item["ingredient"]] += 1 if candidate: candidates.append(candidate) write_jsonl(args.output, candidates) by_reason = defaultdict(int) for audit in audits: if audit["accepted"]: by_reason["accepted"] += 1 else: if audit["row_coverage"] < args.min_row_coverage: by_reason["low_row_coverage"] += 1 if audit["amount_coverage"] < args.min_amount_coverage: by_reason["low_amount_coverage"] += 1 if audit["resolved_components"] < args.min_resolved_components: by_reason["too_few_components"] += 1 summary = { "source": str(args.input), "output": str(args.output), "records_total": len(audits), "records_accepted": len(candidates), "records_rejected": len(audits) - len(candidates), "thresholds": { "min_row_coverage": args.min_row_coverage, "min_amount_coverage": args.min_amount_coverage, "min_resolved_components": args.min_resolved_components, }, "rejection_counts": dict(sorted(by_reason.items())), "top_unresolved": unresolved_counter.most_common(100), "records": audits, } args.audit_output.parent.mkdir(parents=True, exist_ok=True) args.audit_output.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps({k: summary[k] for k in ["records_total", "records_accepted", "records_rejected"]}, indent=2)) if __name__ == "__main__": main()