eac-ledger / internal /normalize /reference.go
runitx's picture
Claude Opus 5.5
eac-ledger: ledger of the EAEU register of conformity documents
2a6941d unverified
Raw History Blame Contribute Delete
4.16 kB
package normalize
import (
"bytes"
"encoding/csv"
"fmt"
"regexp"
"strings"
"unicode"
"eacledger/ref"
)
// Brand is one row of ref/brands.csv.
type Brand struct {
Name string
Country string
Sector string
MfrOnly bool // match manufacturer names only, never free product text
Pattern string
re *regexp.Regexp
}
// Brands finds brand owners in manufacturer names and product text.
type Brands struct {
All []*Brand
byToken map[string][]*Brand
}
// LoadBrands parses the embedded brand table.
func LoadBrands() (*Brands, error) {
rows, err := readCSV(ref.Brands)
if err != nil {
return nil, fmt.Errorf("brands.csv: %w", err)
}
b := &Brands{byToken: map[string][]*Brand{}}
for _, r := range rows {
if len(r) < 5 {
continue
}
re, err := regexp.Compile(`(?:^|[^\p{L}\p{N}])(?:` + r[4] + `)(?:$|[^\p{L}\p{N}])`)
if err != nil {
return nil, fmt.Errorf("brand %s: %w", r[0], err)
}
br := &Brand{Name: r[0], Country: r[1], Sector: r[2], MfrOnly: r[3] == "mfr", Pattern: r[4], re: re}
b.All = append(b.All, br)
for _, t := range firstTokens(r[4]) {
b.byToken[t] = append(b.byToken[t], br)
}
}
return b, nil
}
// firstTokens returns the leading literal word of each top-level
// alternative of a pattern, used to pre-filter texts cheaply.
func firstTokens(pattern string) []string {
var alts []string
depth, start := 0, 0
for i, r := range pattern {
switch r {
case '(':
depth++
case ')':
depth--
case '|':
if depth == 0 {
alts = append(alts, pattern[start:i])
start = i + 1
}
}
}
alts = append(alts, pattern[start:])
seen := map[string]bool{}
var out []string
for _, a := range alts {
var tb strings.Builder
for _, r := range a {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
tb.WriteRune(r)
continue
}
break
}
t := tb.String()
if t != "" && !seen[t] {
seen[t] = true
out = append(out, t)
}
}
return out
}
// Match returns the brands found in text. mfr reports whether text is a
// manufacturer name; brands marked mfr-only are skipped for product text.
func (b *Brands) Match(text string, mfr bool) []*Brand {
if text == "" {
return nil
}
u := Upper(text)
var cands []*Brand
seen := map[*Brand]bool{}
for _, tok := range strings.FieldsFunc(u, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) {
for _, br := range b.byToken[tok] {
if !seen[br] && (mfr || !br.MfrOnly) {
seen[br] = true
cands = append(cands, br)
}
}
}
var out []*Brand
for _, br := range cands {
if br.re.MatchString(u) {
out = append(out, br)
}
}
return out
}
// Reference holds the small lookup tables used while ingesting.
type Reference struct {
Brands *Brands
Group map[string]string // ISO2 -> "eaeu" | "coalition"
CHPTier map[string]string // HS-6 -> tier
CHPDesc map[string]string
Country map[string]string // ISO2 -> English name
}
// LoadReference loads every embedded table.
func LoadReference() (*Reference, error) {
br, err := LoadBrands()
if err != nil {
return nil, err
}
r := &Reference{Brands: br, Group: map[string]string{}, CHPTier: map[string]string{}, CHPDesc: map[string]string{}, Country: map[string]string{}}
rows, err := readCSV(ref.Countries)
if err != nil {
return nil, fmt.Errorf("countries.csv: %w", err)
}
for _, row := range rows {
r.Group[row[0]] = row[2]
r.Country[row[0]] = row[1]
}
rows, err = readCSV(ref.CHPL)
if err != nil {
return nil, fmt.Errorf("chpl.csv: %w", err)
}
for _, row := range rows {
r.CHPTier[row[0]] = row[1]
r.CHPDesc[row[0]] = row[2]
}
return r, nil
}
// Coalition reports whether a country belongs to the export-control coalition.
func (r *Reference) Coalition(iso2 string) bool { return r.Group[iso2] == "coalition" }
// EAEU reports whether a country is an EAEU member.
func (r *Reference) EAEU(iso2 string) bool { return r.Group[iso2] == "eaeu" }
func readCSV(b []byte) ([][]string, error) {
cr := csv.NewReader(bytes.NewReader(b))
cr.FieldsPerRecord = -1
rows, err := cr.ReadAll()
if err != nil {
return nil, err
}
if len(rows) > 0 {
rows = rows[1:] // header
}
return rows, nil
}