package normalize import ( "bytes" "encoding/csv" "fmt" "regexp" "strings" "unicode" "eacledger/ref" ) // Brand is one row of ref/brands.csv. type Brand struct { Name string Country string Sector string MfrOnly bool // match manufacturer names only, never free product text Pattern string re *regexp.Regexp } // Brands finds brand owners in manufacturer names and product text. type Brands struct { All []*Brand byToken map[string][]*Brand } // LoadBrands parses the embedded brand table. func LoadBrands() (*Brands, error) { rows, err := readCSV(ref.Brands) if err != nil { return nil, fmt.Errorf("brands.csv: %w", err) } b := &Brands{byToken: map[string][]*Brand{}} for _, r := range rows { if len(r) < 5 { continue } re, err := regexp.Compile(`(?:^|[^\p{L}\p{N}])(?:` + r[4] + `)(?:$|[^\p{L}\p{N}])`) if err != nil { return nil, fmt.Errorf("brand %s: %w", r[0], err) } br := &Brand{Name: r[0], Country: r[1], Sector: r[2], MfrOnly: r[3] == "mfr", Pattern: r[4], re: re} b.All = append(b.All, br) for _, t := range firstTokens(r[4]) { b.byToken[t] = append(b.byToken[t], br) } } return b, nil } // firstTokens returns the leading literal word of each top-level // alternative of a pattern, used to pre-filter texts cheaply. func firstTokens(pattern string) []string { var alts []string depth, start := 0, 0 for i, r := range pattern { switch r { case '(': depth++ case ')': depth-- case '|': if depth == 0 { alts = append(alts, pattern[start:i]) start = i + 1 } } } alts = append(alts, pattern[start:]) seen := map[string]bool{} var out []string for _, a := range alts { var tb strings.Builder for _, r := range a { if unicode.IsLetter(r) || unicode.IsDigit(r) { tb.WriteRune(r) continue } break } t := tb.String() if t != "" && !seen[t] { seen[t] = true out = append(out, t) } } return out } // Match returns the brands found in text. mfr reports whether text is a // manufacturer name; brands marked mfr-only are skipped for product text. func (b *Brands) Match(text string, mfr bool) []*Brand { if text == "" { return nil } u := Upper(text) var cands []*Brand seen := map[*Brand]bool{} for _, tok := range strings.FieldsFunc(u, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) { for _, br := range b.byToken[tok] { if !seen[br] && (mfr || !br.MfrOnly) { seen[br] = true cands = append(cands, br) } } } var out []*Brand for _, br := range cands { if br.re.MatchString(u) { out = append(out, br) } } return out } // Reference holds the small lookup tables used while ingesting. type Reference struct { Brands *Brands Group map[string]string // ISO2 -> "eaeu" | "coalition" CHPTier map[string]string // HS-6 -> tier CHPDesc map[string]string Country map[string]string // ISO2 -> English name } // LoadReference loads every embedded table. func LoadReference() (*Reference, error) { br, err := LoadBrands() if err != nil { return nil, err } r := &Reference{Brands: br, Group: map[string]string{}, CHPTier: map[string]string{}, CHPDesc: map[string]string{}, Country: map[string]string{}} rows, err := readCSV(ref.Countries) if err != nil { return nil, fmt.Errorf("countries.csv: %w", err) } for _, row := range rows { r.Group[row[0]] = row[2] r.Country[row[0]] = row[1] } rows, err = readCSV(ref.CHPL) if err != nil { return nil, fmt.Errorf("chpl.csv: %w", err) } for _, row := range rows { r.CHPTier[row[0]] = row[1] r.CHPDesc[row[0]] = row[2] } return r, nil } // Coalition reports whether a country belongs to the export-control coalition. func (r *Reference) Coalition(iso2 string) bool { return r.Group[iso2] == "coalition" } // EAEU reports whether a country is an EAEU member. func (r *Reference) EAEU(iso2 string) bool { return r.Group[iso2] == "eaeu" } func readCSV(b []byte) ([][]string, error) { cr := csv.NewReader(bytes.NewReader(b)) cr.FieldsPerRecord = -1 rows, err := cr.ReadAll() if err != nil { return nil, err } if len(rows) > 0 { rows = rows[1:] // header } return rows, nil }