wikiharvest bundle for the Azure containers (public inputs: the undated-marriage pool + sitelinks)

wikiharvest/found.csv.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64
+size 20

wikiharvest/wiki_start_harvest.py

@@ -139,8 +139,55 @@ def sitelinks():
 
 
 # ── phase C: the wikitext harvest ────────────────────────────────────────────────────────────────────────────────
+MONTHS = {
+    "en": "january february march april may june july august september october november december",
+    "de": "januar februar märz april mai juni juli august september oktober november dezember",
+    "fr": "janvier février mars avril mai juin juillet août septembre octobre novembre décembre",
+    "es": "enero febrero marzo abril mayo junio julio agosto septiembre octubre noviembre diciembre",
+    "it": "gennaio febbraio marzo aprile maggio giugno luglio agosto settembre ottobre novembre dicembre",
+    "pt": "janeiro fevereiro março abril maio junho julho agosto setembro outubro novembro dezembro",
+    "nl": "januari februari maart april mei juni juli augustus september oktober november december",
+    "sv": "januari februari mars april maj juni juli augusti september oktober november december",
+    "da": "januar februar marts april maj juni juli august september oktober november december",
+    "fi": "tammikuuta helmikuuta maaliskuuta huhtikuuta toukokuuta kesäkuuta heinäkuuta elokuuta syyskuuta lokakuuta marraskuuta joulukuuta",
+    "pl": "stycznia lutego marca kwietnia maja czerwca lipca sierpnia września października listopada grudnia",
+    "cs": "ledna února března dubna května června července srpna září října listopadu prosince",
+    "ru": "января февраля марта апреля мая июня июля августа сентября октября ноября декабря",
+    "uk": "січня лютого березня квітня травня червня липня серпня вересня жовтня листопада грудня",
+}
+_MTAB = {lang: {m: i + 1 for i, m in enumerate(v.split())} for lang, v in MONTHS.items()}
+
+
+def _full_date(lang, snippet):
+    """A day-precision ISO date inside the snippet, or None: ISO, '11 March 1902' (15 languages' month names,
+    prefix-matched), 'March 11, 1902', ja/zh 1902年3月11日."""
+    iso = re.search(YEAR + r"-(\d\d)-(\d\d)", snippet)
+    if iso:
+        return iso.group(0)
+    m = re.search(YEAR + r"年(\d{1,2})月(\d{1,2})日", snippet)
+    if m:
+        return f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
+    tab = _MTAB.get(lang, {}); tab_en = _MTAB["en"]
+    m = re.search(r"(\d{1,2})\.?\s+([^\W\d_]+),?\s+" + YEAR, snippet, re.U) or re.search(r"([^\W\d_]+)\s+(\d{1,2}),?\s+" + YEAR, snippet, re.U)
+    if m:
+        g = m.groups(); word = (g[1] if g[0].isdigit() else g[0]).lower(); day = int(g[0] if g[0].isdigit() else g[1]); yr = g[2]
+        for t in (tab, tab_en):
+            for name, mon in t.items():
+                if word.startswith(name[:4]) and 1 <= day <= 31:
+                    return f"{yr}-{mon:02d}-{day:02d}"
+    return None
+
+
+def _end_year(snippet, start_year):
+    """The marriage's END year inside the same construct: any later year in the snippet (a {{marriage}} second
+    argument, 'div. 1911', 'ended 1911'); None when the construct names none."""
+    ys = [int(y) for y in re.findall(YEAR, snippet)]
+    later = [y for y in ys if y > start_year]
+    return min(later) if later else None
+
+
 def _find_dates(lang, text, spouse_names):
-    """(start_iso, prec, snippet) for the marriage to this spouse in this wikitext, or None. prec 11 day / 9 year."""
+    """(start_iso, prec, snippet, end_year) for the marriage to this spouse, or None. prec 11 day / 9 year."""
     esc = [re.escape(n) for n in spouse_names if n and len(n) > 2]
     if not esc:
         return None
@@ -151,32 +198,63 @@ def _find_dates(lang, text, spouse_names):
             body = m.group(1)
             if not re.search(name_re, body, re.I):
                 continue
-            iso = re.search(YEAR + r"-(\d\d)-(\d\d)", body)
-            if iso:
-                return iso.group(0), 11, body[:160]
-            MONTHS_EN = {m: i + 1 for i, m in enumerate(("january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"))}
-            d1 = re.search(r"(\d{1,2})\s+([A-Za-z]+)\s+" + YEAR, body) or re.search(r"([A-Za-z]+)\s+(\d{1,2}),\s*" + YEAR, body)
-            if d1:
-                g = d1.groups(); mon = MONTHS_EN.get((g[1] if g[0].isdigit() else g[0]).lower()); day = int(g[0] if g[0].isdigit() else g[1]); yr4 = g[2]
-                if mon:
-                    return f"{yr4}-{mon:02d}-{day:02d}", 11, body[:160]
+            fd = _full_date(lang, body)
+            if fd:
+                return fd, 11, body[:160], _end_year(body, int(fd[:4]))
             yr = re.search(YEAR, body)
             if yr:
-                return yr.group(0) + "-00-00", 9, body[:160]
+                return yr.group(0) + "-00-00", 9, body[:160], _end_year(body, int(yr.group(0)))
     # 2. the spouse parameter's value line: |spouse = [[Name]] (m. 1903; div 1919) / ⚭ 1903 / с 1903
     for par in SPOUSE_PARAMS.get(lang, []) + SPOUSE_PARAMS.get("en", []):
         for m in re.finditer(r"\|\s*" + re.escape(par) + r"\s*=([^\n]*?" + name_re + r"[^\n]*)", text, re.I):
             line = m.group(1)
-            yr = re.search(YEAR, line)
-            if yr:
-                return yr.group(0) + "-00-00", 9, line[:160]
+            # a line can list several spouses -- "A (1890-1900); B (1902-1911)" -- the year must come from OUR
+            # spouse's SEGMENT, never the first year in the line
+            for seg in re.split(r"(?<=\))\s*;|<br\s*/?>|\}\}\s*\{\{", line):   # a spouse separator is ");" — semicolons INSIDE a parenthetical stay
+                if not re.search(name_re, seg, re.I):
+                    continue
+                fd = _full_date(lang, seg)
+                if fd:
+                    return fd, 11, seg[:160], _end_year(seg, int(fd[:4]))
+                yr = re.search(YEAR, seg)
+                if yr:
+                    return yr.group(0) + "-00-00", 9, seg[:160], _end_year(seg, int(yr.group(0)))
     # 3. a parenthetical right after the spouse's name, REQUIRING a marriage marker — "(1848–1919)" is a lifespan
-    MARK = {"en": r"m\.|marr", "de": r"⚭|verh", "fr": r"mari|ép", "es": r"matr|casad", "it": r"spos", "ru": r"брак|с\s", "uk": r"шлюб|з\s", "pl": r"ślub|od\s", "pt": r"casad", "nl": r"getr", "sv": r"gift", "da": r"gift", "fi": r"avio", "cs": r"sňat|od\s", "hu": r"házas", "tr": r"evl", "fa": r"ازدواج", "ar": r"تزوج|زواج", "ja": r"結婚", "zh": r"结婚|結婚", "hy": r"ամուսն"}
-    mark = MARK.get(lang, r"⚭|m\.")
+    MARK = {"en": r"m\.|marr", "de": r"⚭|∞|verh", "fr": r"mari|ép", "es": r"matr|casad", "it": r"spos", "ru": r"брак|с\s", "uk": r"шлюб|з\s", "pl": r"ślub|od\s", "pt": r"casad", "nl": r"getr", "sv": r"gift", "da": r"gift", "fi": r"avio", "cs": r"sňat|od\s", "hu": r"házas", "tr": r"evl", "fa": r"ازدواج", "ar": r"تزوج|زواج", "ja": r"結婚", "zh": r"结婚|結婚", "hy": r"ամուսն"}
+    mark = MARK.get(lang, r"⚭|∞|m\.")
     m = re.search(name_re + r"[^\n(]{0,40}\((?:[^)]{0,40}?(?:" + mark + r")[^)]{0,40}?)" + YEAR + r"[^)]*\)", text, re.I)
     if m:
         yr = re.search(YEAR, m.group(0))
-        return yr.group(0) + "-00-00", 9, m.group(0)[:160]
+        return yr.group(0) + "-00-00", 9, m.group(0)[:160], _end_year(m.group(0), int(yr.group(0)))
+    # 3b. the genealogy signs need no parenthetical: "[[Alma]] ⚭ 1902" / "∞ 1902 [[Alma]]"
+    for pat in (name_re + r"[^\n]{0,15}[⚭∞][^\n]{0,15}" + YEAR, r"[⚭∞][^\n]{0,15}" + YEAR + r"[^\n]{0,25}" + name_re, YEAR + r"[^\n]{0,15}[⚭∞][^\n]{0,25}" + name_re):
+        m = re.search(pat, text, re.I)
+        if m:
+            yr = re.search(YEAR, m.group(0))
+            return yr.group(0) + "-00-00", 9, m.group(0)[:160], _end_year(m.group(0), int(yr.group(0)))
+    # 4. PROSE (the wikis that keep marriage in running text — dewiki rejects person infoboxes altogether):
+    #    "heiratete [am 9. März] 1902 [[Alma]]" / "épousa [[Alma]] en 1902" / "женился на [[Альме]] в 1902" —
+    #    a marriage VERB, the spouse and a year within one clause, either order; the plausibility gate downstream
+    #    still rejects any year that cannot be a wedding
+    VERB = {"de": r"heiratete|ehelichte|verm[äa]hlte|schloss.{0,20}Ehe", "fr": r"épous[ae]|se mari[ae]|mariage avec", "es": r"se cas[óo]|contrajo matrimonio",
+            "it": r"spos[òo]", "pt": r"casou(?:-se)?", "ru": r"женился|вышла замуж|вступил[аи]? в брак|обвенчал", "uk": r"одружився|вийшла заміж",
+            "pl": r"poślubił|ożenił|wyszła za", "nl": r"trouwde|huwde", "sv": r"gifte sig", "da": r"giftede sig", "fi": r"avioitui|meni naimisiin",
+            "cs": r"oženil se|vdala se|vzal si", "hu": r"feleségül vette|házasságot kötött", "tr": r"evlendi", "ja": r"結婚", "zh": r"结婚|結婚",
+            "fa": r"ازدواج کرد", "ar": r"تزوج", "hy": r"ամուսնացավ|ամուսնացել", "en": r"married"}
+    verb = VERB.get(lang)
+    if verb:
+        win = r"[^\n;]{0,90}"          # periods allowed: German ordinal dates ("9. März 1902") sit inside the clause
+        V = r"(?:" + verb + r")"
+        for pat in (V + win + name_re + win + YEAR, V + win + YEAR + win + name_re, name_re + win + V + win + YEAR,
+                    YEAR + win + V + win + name_re, name_re + win + YEAR + win + V, YEAR + win + name_re + win + V):   # all six orders
+            m = re.search(pat, text, re.I)
+            if m:
+                clause = text[max(0, m.start() - 24):m.end()]        # a day often precedes the matched year ("9 марта 1902 ...")
+                fd = _full_date(lang, clause)
+                if fd:
+                    return fd, 11, clause[:160], _end_year(clause, int(fd[:4]))
+                yr = re.search(YEAR, m.group(0))
+                return yr.group(0) + "-00-00", 9, m.group(0)[:160], _end_year(m.group(0), int(yr.group(0)))
     return None
 
 
@@ -217,7 +295,7 @@ def harvest(langs=None):
                     continue
                 hit = _find_dates(lang, text, names)
                 if hit:
-                    st, prec, snip = hit; sy = int(st[:4])
+                    st, prec, snip, endy = hit; sy = int(st[:4])
                     by = [int(r[k][:4]) for k in ("adob", "bdob") if r[k][:4].isdigit()]
                     ey = [int(r[k][:4]) for k in ("end", "adeath", "bdeath") if r[k][:4].isdigit()]
                     # a wedding before either 14th birthday, or after the marriage's own end, is a misread — dropped
@@ -225,7 +303,7 @@ def harvest(langs=None):
                         continue
                     if ey and sy > min(ey):
                         continue
-                    w.writerow([r["a"], r["b"], st, prec, lang, subj, snip.replace("\n", " ")]); got += 1
+                    w.writerow([r["a"], r["b"], st, prec, lang, subj, snip.replace("\n", " "), endy if endy is not None else ""]); got += 1
             if i % 400 == 0 and i:
                 f.flush(); log(f"  {lang}: {i:,}/{len(jobs):,} read · {got:,} dates found")
             time.sleep(0.6)