Download internal/normalize/reference.go from abr00kx/eac-ledger: direct link, hf CLI and curl.
- Browser
- Download file 4.16 kB
-
https://huggingface.co/abr00kx/eac-ledger/resolve/main/internal/normalize/reference.go
- Command line
-
hf download hf://abr00kx/eac-ledger/internal/normalize/reference.go
-
curl -L -o reference.go https://huggingface.co/abr00kx/eac-ledger/resolve/main/internal/normalize/reference.go
4.16 kB
| package normalize | |
| import ( | |
| "bytes" | |
| "encoding/csv" | |
| "fmt" | |
| "regexp" | |
| "strings" | |
| "unicode" | |
| "eacledger/ref" | |
| ) | |
| // Brand is one row of ref/brands.csv. | |
| type Brand struct { | |
| Name string | |
| Country string | |
| Sector string | |
| MfrOnly bool // match manufacturer names only, never free product text | |
| Pattern string | |
| re *regexp.Regexp | |
| } | |
| // Brands finds brand owners in manufacturer names and product text. | |
| type Brands struct { | |
| All []*Brand | |
| byToken map[string][]*Brand | |
| } | |
| // LoadBrands parses the embedded brand table. | |
| func LoadBrands() (*Brands, error) { | |
| rows, err := readCSV(ref.Brands) | |
| if err != nil { | |
| return nil, fmt.Errorf("brands.csv: %w", err) | |
| } | |
| b := &Brands{byToken: map[string][]*Brand{}} | |
| for _, r := range rows { | |
| if len(r) < 5 { | |
| continue | |
| } | |
| re, err := regexp.Compile(`(?:^|[^\p{L}\p{N}])(?:` + r[4] + `)(?:$|[^\p{L}\p{N}])`) | |
| if err != nil { | |
| return nil, fmt.Errorf("brand %s: %w", r[0], err) | |
| } | |
| br := &Brand{Name: r[0], Country: r[1], Sector: r[2], MfrOnly: r[3] == "mfr", Pattern: r[4], re: re} | |
| b.All = append(b.All, br) | |
| for _, t := range firstTokens(r[4]) { | |
| b.byToken[t] = append(b.byToken[t], br) | |
| } | |
| } | |
| return b, nil | |
| } | |
| // firstTokens returns the leading literal word of each top-level | |
| // alternative of a pattern, used to pre-filter texts cheaply. | |
| func firstTokens(pattern string) []string { | |
| var alts []string | |
| depth, start := 0, 0 | |
| for i, r := range pattern { | |
| switch r { | |
| case '(': | |
| depth++ | |
| case ')': | |
| depth-- | |
| case '|': | |
| if depth == 0 { | |
| alts = append(alts, pattern[start:i]) | |
| start = i + 1 | |
| } | |
| } | |
| } | |
| alts = append(alts, pattern[start:]) | |
| seen := map[string]bool{} | |
| var out []string | |
| for _, a := range alts { | |
| var tb strings.Builder | |
| for _, r := range a { | |
| if unicode.IsLetter(r) || unicode.IsDigit(r) { | |
| tb.WriteRune(r) | |
| continue | |
| } | |
| break | |
| } | |
| t := tb.String() | |
| if t != "" && !seen[t] { | |
| seen[t] = true | |
| out = append(out, t) | |
| } | |
| } | |
| return out | |
| } | |
| // Match returns the brands found in text. mfr reports whether text is a | |
| // manufacturer name; brands marked mfr-only are skipped for product text. | |
| func (b *Brands) Match(text string, mfr bool) []*Brand { | |
| if text == "" { | |
| return nil | |
| } | |
| u := Upper(text) | |
| var cands []*Brand | |
| seen := map[*Brand]bool{} | |
| for _, tok := range strings.FieldsFunc(u, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) { | |
| for _, br := range b.byToken[tok] { | |
| if !seen[br] && (mfr || !br.MfrOnly) { | |
| seen[br] = true | |
| cands = append(cands, br) | |
| } | |
| } | |
| } | |
| var out []*Brand | |
| for _, br := range cands { | |
| if br.re.MatchString(u) { | |
| out = append(out, br) | |
| } | |
| } | |
| return out | |
| } | |
| // Reference holds the small lookup tables used while ingesting. | |
| type Reference struct { | |
| Brands *Brands | |
| Group map[string]string // ISO2 -> "eaeu" | "coalition" | |
| CHPTier map[string]string // HS-6 -> tier | |
| CHPDesc map[string]string | |
| Country map[string]string // ISO2 -> English name | |
| } | |
| // LoadReference loads every embedded table. | |
| func LoadReference() (*Reference, error) { | |
| br, err := LoadBrands() | |
| if err != nil { | |
| return nil, err | |
| } | |
| r := &Reference{Brands: br, Group: map[string]string{}, CHPTier: map[string]string{}, CHPDesc: map[string]string{}, Country: map[string]string{}} | |
| rows, err := readCSV(ref.Countries) | |
| if err != nil { | |
| return nil, fmt.Errorf("countries.csv: %w", err) | |
| } | |
| for _, row := range rows { | |
| r.Group[row[0]] = row[2] | |
| r.Country[row[0]] = row[1] | |
| } | |
| rows, err = readCSV(ref.CHPL) | |
| if err != nil { | |
| return nil, fmt.Errorf("chpl.csv: %w", err) | |
| } | |
| for _, row := range rows { | |
| r.CHPTier[row[0]] = row[1] | |
| r.CHPDesc[row[0]] = row[2] | |
| } | |
| return r, nil | |
| } | |
| // Coalition reports whether a country belongs to the export-control coalition. | |
| func (r *Reference) Coalition(iso2 string) bool { return r.Group[iso2] == "coalition" } | |
| // EAEU reports whether a country is an EAEU member. | |
| func (r *Reference) EAEU(iso2 string) bool { return r.Group[iso2] == "eaeu" } | |
| func readCSV(b []byte) ([][]string, error) { | |
| cr := csv.NewReader(bytes.NewReader(b)) | |
| cr.FieldsPerRecord = -1 | |
| rows, err := cr.ReadAll() | |
| if err != nil { | |
| return nil, err | |
| } | |
| if len(rows) > 0 { | |
| rows = rows[1:] // header | |
| } | |
| return rows, nil | |
| } | |