Download internal/normalize/normalize.go from abr00kx/eac-ledger: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/abr00kx/eac-ledger/resolve/main/internal/normalize/normalize.go
- Command line
-
hf download hf://abr00kx/eac-ledger/internal/normalize/normalize.go
-
curl -L -o normalize.go https://huggingface.co/abr00kx/eac-ledger/resolve/main/internal/normalize/normalize.go
10.1 kB
| // Package normalize turns raw register text into comparable keys: company | |
| // names without legal forms, HS codes, hashed contact keys and Russian | |
| // federal subjects. | |
| package normalize | |
| import ( | |
| "crypto/hmac" | |
| "crypto/sha256" | |
| "encoding/hex" | |
| "regexp" | |
| "strings" | |
| "unicode" | |
| ) | |
| var ( | |
| quotes = strings.NewReplacer("«", " ", "»", " ", "\"", " ", "'", " ", "“", " ", "”", " ", "„", " ", "`", " ", "Ё", "Е", "ё", "е") | |
| dotted = regexp.MustCompile(`\b([A-ZА-Я])\.`) | |
| nonAlnum = regexp.MustCompile(`[^\p{L}\p{N}&+]+`) | |
| ) | |
| // legalPhrases are multi-word legal forms, removed before tokenising. | |
| var legalPhrases = []string{ | |
| "ОБЩЕСТВО С ОГРАНИЧЕННОЙ ОТВЕТСТВЕННОСТЬЮ", "ОБЩЕСТВОМ С ОГРАНИЧЕННОЙ ОТВЕТСТВЕННОСТЬЮ", | |
| "ТОВАРИЩЕСТВО С ОГРАНИЧЕННОЙ ОТВЕТСТВЕННОСТЬЮ", "ОБЩЕСТВО С ДОПОЛНИТЕЛЬНОЙ ОТВЕТСТВЕННОСТЬЮ", | |
| "НЕПУБЛИЧНОЕ АКЦИОНЕРНОЕ ОБЩЕСТВО", "ПУБЛИЧНОЕ АКЦИОНЕРНОЕ ОБЩЕСТВО", | |
| "ЗАКРЫТОЕ АКЦИОНЕРНОЕ ОБЩЕСТВО", "ОТКРЫТОЕ АКЦИОНЕРНОЕ ОБЩЕСТВО", "АКЦИОНЕРНОЕ ОБЩЕСТВО", | |
| "ИНДИВИДУАЛЬНЫЙ ПРЕДПРИНИМАТЕЛЬ", "ЧАСТНОЕ ПРОИЗВОДСТВЕННОЕ УНИТАРНОЕ ПРЕДПРИЯТИЕ", | |
| "ЧАСТНОЕ ТОРГОВОЕ УНИТАРНОЕ ПРЕДПРИЯТИЕ", "ЧАСТНОЕ УНИТАРНОЕ ПРЕДПРИЯТИЕ", | |
| "ФЕДЕРАЛЬНОЕ ГОСУДАРСТВЕННОЕ УНИТАРНОЕ ПРЕДПРИЯТИЕ", "ГОСУДАРСТВЕННОЕ УНИТАРНОЕ ПРЕДПРИЯТИЕ", | |
| "РЕСПУБЛИКАНСКОЕ УНИТАРНОЕ ПРЕДПРИЯТИЕ", "МУНИЦИПАЛЬНОЕ УНИТАРНОЕ ПРЕДПРИЯТИЕ", | |
| "СОВМЕСТНОЕ ПРЕДПРИЯТИЕ", "ПРОИЗВОДСТВЕННЫЙ КООПЕРАТИВ", "ЖЕКЕ КӘСІПКЕР", | |
| "ЖАУАПКЕРШІЛІГІ ШЕКТЕУЛІ СЕРІКТЕСТІГІ", "ЖАУАПКЕРШІЛІГІ ШЕКТЕУЛІ СЕРІКТЕСТІК", | |
| "LIMITED LIABILITY COMPANY", "LIMITED LIABILITY PARTNERSHIP", "JOINT STOCK COMPANY", | |
| "PUBLIC LIMITED COMPANY", "PRIVATE LIMITED", "CO LTD", "CO KG", "GMBH & CO", | |
| "SP Z O O", "SP ZOO", "S DE RL DE CV", "SA DE CV", | |
| } | |
| // legalTokens are single-token legal forms and filler words. | |
| var legalTokens = map[string]bool{ | |
| "ООО": true, "ОАО": true, "ЗАО": true, "ПАО": true, "АО": true, "НАО": true, "ИП": true, "ТОО": true, | |
| "ОДО": true, "ЧУП": true, "ЧТУП": true, "ЧПУП": true, "СООО": true, "ИООО": true, "УП": true, | |
| "ФГУП": true, "ГУП": true, "МУП": true, "КФХ": true, "ЖШС": true, "АҚ": true, "ОсОО": true, "ОСОО": true, | |
| "GMBH": true, "MBH": true, "AG": true, "KG": true, "KGAA": true, "SE": true, "LTD": true, "LIMITED": true, "LLC": true, | |
| "INC": true, "INCORPORATED": true, "CORP": true, "CORPORATION": true, "CO": true, "COMPANY": true, "PLC": true, | |
| "LLP": true, "LP": true, "SA": true, "SAS": true, "SARL": true, "SRL": true, "SPA": true, "SL": true, "SLU": true, | |
| "BV": true, "NV": true, "OY": true, "OYJ": true, "AB": true, "AS": true, "ASA": true, "APS": true, "A/S": true, | |
| "KFT": true, "ZRT": true, "SRO": true, "DOO": true, "DD": true, "PTE": true, "PTY": true, "BHD": true, "SDN": true, | |
| "JSC": true, "OJSC": true, "CJSC": true, "PJSC": true, "OOO": true, "ZAO": true, "OAO": true, "TOO": true, | |
| "SC": true, "SCS": true, "KK": true, "THE": true, | |
| } | |
| // Name returns a comparison key for a company name: upper case, quotes and | |
| // legal forms removed, punctuation collapsed. | |
| func Name(s string) string { | |
| s = strings.ToUpper(quotes.Replace(s)) | |
| s = dotted.ReplaceAllString(s, "$1") | |
| s = nonAlnum.ReplaceAllString(s, " ") | |
| s = " " + strings.Join(strings.Fields(s), " ") + " " | |
| for _, p := range legalPhrases { | |
| s = strings.ReplaceAll(s, " "+p+" ", " ") | |
| } | |
| toks := strings.Fields(s) | |
| out := toks[:0] | |
| for _, t := range toks { | |
| if !legalTokens[t] { | |
| out = append(out, t) | |
| } | |
| } | |
| if len(out) == 0 { // the name was nothing but a legal form; keep it | |
| return strings.Join(strings.Fields(s), " ") | |
| } | |
| return strings.Join(out, " ") | |
| } | |
| // Upper upper-cases text for pattern matching and folds Ё to Е. | |
| func Upper(s string) string { | |
| return strings.ToUpper(quotes.Replace(s)) | |
| } | |
| var ( | |
| personPrefix = regexp.MustCompile(`^\s*(ИП|И\.П\.|ИНДИВИДУАЛЬНЫЙ ПРЕДПРИНИМАТЕЛЬ|ЖЕКЕ КӘСІПКЕР|КФХ|ГЛАВА КФХ|ГЛАВА КРЕСТЬЯНСКОГО|КРЕСТЬЯНСКОЕ \(ФЕРМЕРСКОЕ\) ХОЗЯЙСТВО|КРЕСТЬЯНСКОЕ ФЕРМЕРСКОЕ ХОЗЯЙСТВО)[\s:]`) | |
| // Surname Name Patronymic, as Russian, Belarusian and Central Asian | |
| // registers write natural persons. | |
| fullName = regexp.MustCompile(`^[А-ЯЁҚҒҮҰӘӨҺІ][А-ЯЁҚҒҮҰӘӨҺІа-яёқғүұәөһі-]+\s+[А-ЯЁҚҒҮҰӘӨҺІ][а-яёқғүұәөһі-]+\s+[А-ЯЁҚҒҮҰӘӨҺІ][а-яёқғүұәөһі]+(?:(?:ВИЧ|ВНА|ИЧНА|ИНИЧНА|КЫЗЫ|ОГЛЫ|УЛЫ|УУЛУ|вич|вна|ична|инична|кызы|оглы|улы|уулу)|\s+(?:КЫЗЫ|ОГЛЫ|УЛЫ|УУЛУ|кызы|оглы|улы|уулу))$`) | |
| // Surname I.O. | |
| initials = regexp.MustCompile(`^[А-ЯЁ][А-ЯЁа-яё-]+\s+[А-ЯЁ]\.\s?[А-ЯЁ]\.?$`) | |
| ) | |
| // NaturalPerson reports whether an applicant is a natural person: an | |
| // individual entrepreneur, the head of a farm household, or an individual | |
| // named without any legal form. Their names and addresses are personal | |
| // data and are withheld from published output. | |
| func NaturalPerson(name, typeName, idKind string) bool { | |
| if idKind == "02" { // ОГРНИП | |
| return true | |
| } | |
| t := strings.ToUpper(typeName) | |
| if strings.Contains(t, "ПРЕДПРИНИМАТЕЛ") || strings.Contains(t, "КӘСІПКЕР") || strings.Contains(t, "КРЕСТЬЯНСК") || strings.Contains(t, "ФЕРМЕРСК") { | |
| return true | |
| } | |
| n := strings.TrimSpace(quotes.Replace(name)) | |
| n = strings.Join(strings.Fields(n), " ") | |
| if personPrefix.MatchString(strings.ToUpper(n) + " ") { | |
| return true | |
| } | |
| return fullName.MatchString(n) || initials.MatchString(n) | |
| } | |
| // HS splits a commodity code field into normalised codes of 4 to 10 digits. | |
| func HS(raw string) []string { | |
| var out []string | |
| for _, part := range strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ';' || r == '\n' }) { | |
| var b strings.Builder | |
| for _, r := range part { | |
| if r >= '0' && r <= '9' { | |
| b.WriteRune(r) | |
| } | |
| } | |
| d := b.String() | |
| if len(d) > 10 { | |
| d = d[:10] | |
| } | |
| if len(d) >= 4 { | |
| out = append(out, d) | |
| } | |
| } | |
| return out | |
| } | |
| // Contact normalises a phone number or e-mail address for hashing. | |
| // It returns "" for values too short to identify anything. | |
| func Contact(channel, v string) string { | |
| v = strings.TrimSpace(v) | |
| switch strings.ToUpper(channel) { | |
| case "EM": | |
| v = strings.ToLower(v) | |
| if !strings.Contains(v, "@") { | |
| return "" | |
| } | |
| return v | |
| case "TE", "FX": | |
| var b strings.Builder | |
| for _, r := range v { | |
| if unicode.IsDigit(r) { | |
| b.WriteRune(r) | |
| } | |
| } | |
| d := b.String() | |
| if len(d) == 11 && d[0] == '8' { | |
| d = "7" + d[1:] | |
| } | |
| if len(d) == 10 && d[0] == '9' { // Russian mobile written without the country code | |
| d = "7" + d | |
| } | |
| if len(d) < 7 { | |
| return "" | |
| } | |
| return d | |
| default: | |
| v = strings.ToLower(strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(v, "https://"), "http://"), "/")) | |
| v = strings.TrimPrefix(v, "www.") | |
| if len(v) < 4 { | |
| return "" | |
| } | |
| return v | |
| } | |
| } | |
| // SplitContacts splits a contact field that holds several values. | |
| func SplitContacts(v string) []string { | |
| return strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ';' }) | |
| } | |
| // Hasher derives stable, non-reversible keys from contact values and | |
| // personal names using a secret key that never leaves the operator. | |
| // Without the key a phone number cannot be recovered by enumeration. | |
| type Hasher struct{ key []byte } | |
| // NewHasher returns a Hasher for key. | |
| func NewHasher(key []byte) *Hasher { return &Hasher{key: key} } | |
| // Sum returns a 16-hex-character keyed hash of v. | |
| func (h *Hasher) Sum(v string) string { | |
| m := hmac.New(sha256.New, h.key) | |
| m.Write([]byte(v)) | |
| return hex.EncodeToString(m.Sum(nil))[:16] | |
| } | |
| // Truncate cuts s to at most n runes. | |
| func Truncate(s string, n int) string { | |
| if len(s) <= n { | |
| return s | |
| } | |
| r := []rune(s) | |
| if len(r) <= n { | |
| return s | |
| } | |
| return string(r[:n]) | |
| } | |
| const cyrU = `А-ЯЁҚҒҮҰӘӨҺІ` | |
| const cyrL = `а-яёқғүұәөһі` | |
| var personInText = []*regexp.Regexp{ | |
| // "ИП Иванов Иван Иванович", "индивидуальный предприниматель Петрова А.С." | |
| regexp.MustCompile(`(^|[^` + cyrU + cyrL + `A-Za-z0-9])(?i:ИП|И\.П\.|индивидуальн(?:ый|ого|ым|ому) предпринимател[ьяюем]+|глав[аы] (?:КФХ|крестьянского \(фермерского\) хозяйства))\s*[:«"]?\s*[` + cyrU + `][` + cyrL + `-]+(?:\s+[` + cyrU + `][` + cyrL + `]+|\s*[` + cyrU + `]\.){0,2}`), | |
| // Surname Name Patronymic | |
| regexp.MustCompile(`[` + cyrU + `][` + cyrL + `-]+\s+[` + cyrU + `][` + cyrL + `]+\s+[` + cyrU + `][` + cyrL + `]+(?:вич|вна|ична|инична)(?:\s+(?:оглы|кызы|улы|уулу))?`), | |
| // Surname I.O. and I.O. Surname | |
| regexp.MustCompile(`[` + cyrU + `][` + cyrL + `]{2,}(?:\s+|\s*,\s*)[` + cyrU + `]\.\s?[` + cyrU + `]\.`), | |
| regexp.MustCompile(`[` + cyrU + `]\.\s?[` + cyrU + `]\.\s?[` + cyrU + `][` + cyrL + `]{2,}`), | |
| } | |
| // ScrubPersons replaces personal names in free text with "[лицо]". | |
| // It is deliberately aggressive: free text in the register is about | |
| // products, and a dropped surname costs less than a published one. | |
| func ScrubPersons(s string) string { | |
| if s == "" { | |
| return s | |
| } | |
| s = personInText[0].ReplaceAllString(s, "${1}[лицо]") | |
| for _, re := range personInText[1:] { | |
| s = re.ReplaceAllString(s, "[лицо]") | |
| } | |
| return s | |
| } | |