File size: 4,160 Bytes
2a6941d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
package normalize

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"regexp"
	"strings"
	"unicode"

	"eacledger/ref"
)

// Brand is one row of ref/brands.csv.
type Brand struct {
	Name    string
	Country string
	Sector  string
	MfrOnly bool // match manufacturer names only, never free product text
	Pattern string
	re      *regexp.Regexp
}

// Brands finds brand owners in manufacturer names and product text.
type Brands struct {
	All     []*Brand
	byToken map[string][]*Brand
}

// LoadBrands parses the embedded brand table.
func LoadBrands() (*Brands, error) {
	rows, err := readCSV(ref.Brands)
	if err != nil {
		return nil, fmt.Errorf("brands.csv: %w", err)
	}
	b := &Brands{byToken: map[string][]*Brand{}}
	for _, r := range rows {
		if len(r) < 5 {
			continue
		}
		re, err := regexp.Compile(`(?:^|[^\p{L}\p{N}])(?:` + r[4] + `)(?:$|[^\p{L}\p{N}])`)
		if err != nil {
			return nil, fmt.Errorf("brand %s: %w", r[0], err)
		}
		br := &Brand{Name: r[0], Country: r[1], Sector: r[2], MfrOnly: r[3] == "mfr", Pattern: r[4], re: re}
		b.All = append(b.All, br)
		for _, t := range firstTokens(r[4]) {
			b.byToken[t] = append(b.byToken[t], br)
		}
	}
	return b, nil
}

// firstTokens returns the leading literal word of each top-level
// alternative of a pattern, used to pre-filter texts cheaply.
func firstTokens(pattern string) []string {
	var alts []string
	depth, start := 0, 0
	for i, r := range pattern {
		switch r {
		case '(':
			depth++
		case ')':
			depth--
		case '|':
			if depth == 0 {
				alts = append(alts, pattern[start:i])
				start = i + 1
			}
		}
	}
	alts = append(alts, pattern[start:])
	seen := map[string]bool{}
	var out []string
	for _, a := range alts {
		var tb strings.Builder
		for _, r := range a {
			if unicode.IsLetter(r) || unicode.IsDigit(r) {
				tb.WriteRune(r)
				continue
			}
			break
		}
		t := tb.String()
		if t != "" && !seen[t] {
			seen[t] = true
			out = append(out, t)
		}
	}
	return out
}

// Match returns the brands found in text. mfr reports whether text is a
// manufacturer name; brands marked mfr-only are skipped for product text.
func (b *Brands) Match(text string, mfr bool) []*Brand {
	if text == "" {
		return nil
	}
	u := Upper(text)
	var cands []*Brand
	seen := map[*Brand]bool{}
	for _, tok := range strings.FieldsFunc(u, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) {
		for _, br := range b.byToken[tok] {
			if !seen[br] && (mfr || !br.MfrOnly) {
				seen[br] = true
				cands = append(cands, br)
			}
		}
	}
	var out []*Brand
	for _, br := range cands {
		if br.re.MatchString(u) {
			out = append(out, br)
		}
	}
	return out
}

// Reference holds the small lookup tables used while ingesting.
type Reference struct {
	Brands  *Brands
	Group   map[string]string // ISO2 -> "eaeu" | "coalition"
	CHPTier map[string]string // HS-6 -> tier
	CHPDesc map[string]string
	Country map[string]string // ISO2 -> English name
}

// LoadReference loads every embedded table.
func LoadReference() (*Reference, error) {
	br, err := LoadBrands()
	if err != nil {
		return nil, err
	}
	r := &Reference{Brands: br, Group: map[string]string{}, CHPTier: map[string]string{}, CHPDesc: map[string]string{}, Country: map[string]string{}}
	rows, err := readCSV(ref.Countries)
	if err != nil {
		return nil, fmt.Errorf("countries.csv: %w", err)
	}
	for _, row := range rows {
		r.Group[row[0]] = row[2]
		r.Country[row[0]] = row[1]
	}
	rows, err = readCSV(ref.CHPL)
	if err != nil {
		return nil, fmt.Errorf("chpl.csv: %w", err)
	}
	for _, row := range rows {
		r.CHPTier[row[0]] = row[1]
		r.CHPDesc[row[0]] = row[2]
	}
	return r, nil
}

// Coalition reports whether a country belongs to the export-control coalition.
func (r *Reference) Coalition(iso2 string) bool { return r.Group[iso2] == "coalition" }

// EAEU reports whether a country is an EAEU member.
func (r *Reference) EAEU(iso2 string) bool { return r.Group[iso2] == "eaeu" }

func readCSV(b []byte) ([][]string, error) {
	cr := csv.NewReader(bytes.NewReader(b))
	cr.FieldsPerRecord = -1
	rows, err := cr.ReadAll()
	if err != nil {
		return nil, err
	}
	if len(rows) > 0 {
		rows = rows[1:] // header
	}
	return rows, nil
}