Cleaning up

.ipynb_checkpoints/README-checkpoint.md

@@ -1,40 +0,0 @@
-# Test
-
-- Train: hela den historiska maskinparsade korpusen `alanev52/Diachronic_Treebanks_DigPhil/stanza_results` plus alla fem svenska trädbanker från ud, och även de från bokmål
-- Dev: 10% av guld (`alanev52/Diachronic_Treebanks_DigPhil/stanza_results/validated`)
-- Test: 90% av guld
-
-<!-- tmux attach -t stanza ****-->
-
-Work flow:
-
-```
-python prepare-train-val-test.py
-
-source scripts/config_alvis.sh
-
-python -m stanza.utils.datasets.prepare_depparse_treebank UD_Swedish-diachronic --wordvec_pretrain_file /cephyr/users/cleland/Alvis/stanza_resources/sv/pretrain/diachronic.pt
-
-python -m stanza.utils.training.run_depparse UD_Swedish-diachronic --wordvec_pretrain_file /cephyr/users/cleland/Alvis/stanza_resources/sv/pretrain/diachronic.pt --batch_size 32 --dropout 0.33
-```
-
-## Pretrained vectors
-
-We use the incremental vectors up until 1880 from Henchen & Tahmasebi 2021.
-
-Jag konverterade först kubhist2-vektorerna från gensim fasttext .ft till en vanlig textfil med gensims pythonpaket, sedan använde jag stanzas konverterare till .pt:
-
-```
-from stanza.models.common.pretrain import Pretrain
-pt = Pretrain("foo.pt", "new_vectors.txt")
-pt.load()
-```
-
-Resultatet finns komprimerat i `diachronic.pt.xz`.
-
-## References
-
-**Hengchen, Simon & Tahmasebi, Nina. (2021).**  
-*A collection of Swedish diachronic word embedding models trained on historical newspaper data.*  
-**Journal of Open Humanities Data**, 7(2), 1–7.  
-https://doi.org/10.5334/johd.22

.ipynb_checkpoints/prepare-train-val-test-checkpoint.py

@@ -1,246 +0,0 @@
-#!/usr/bin/env python3
-import random
-from pathlib import Path
-from collections import defaultdict
-
-# ============================================================
-# BASE PATHS
-# ============================================================
-BASE = Path("/mimer/NOBACKUP/groups/dionysus/cleland/stanza-digphil").resolve()
-
-SVENSKA_PROJEKT = BASE / "ud-treebanks-sv"
-NORSKA_PROJEKT = BASE / "ud-treebanks-bm"
-DANSKA_PROJEKT = BASE / "ud-treebanks-dk"
-
-DIGPHIL_MACHINE = BASE / "alanev_raw_files/diachron"
-DIGPHIL_GOLD = BASE / "alanev_raw_files/diachron-validated"
-
-OUTPUT_TRAIN = BASE / "sv_diachronic-ud-train.conllu"
-OUTPUT_DEV   = BASE / "sv_diachronic-ud-dev.conllu"
-OUTPUT_TEST  = BASE / "sv_diachronic-ud-test.conllu"
-
-random.seed(1337)
-
-# ============================================================
-# BASIC CONLLU HELPERS
-# ============================================================
-def read_conllu(path: Path):
-    text = path.read_text(encoding="utf-8").strip()
-    return [] if not text else text.split("\n\n")
-
-def extract_sent_id(block: str) -> str | None:
-    for line in block.split("\n"):
-        if line.startswith("# sent_id"):
-            parts = line.split("=", 1)
-            if len(parts) == 2:
-                return parts[1].strip()
-            return line.split("# sent_id", 1)[1].strip()
-    return None
-
-def write_conllu(path: Path, sentences):
-    with path.open("w", encoding="utf-8") as f:
-        for s in sentences:
-            f.write(s.strip() + "\n\n")
-
-def load_from_treebank_dir(directory: Path):
-    collected = []
-    for path in directory.rglob("*.conllu"):
-        print(f"Reading: {path}")
-        collected.extend(read_conllu(path))
-    return collected
-
-# ============================================================
-# CoNLL-U VALIDATOR (unchanged logic, adapted for in-memory)
-# ============================================================
-class CoNLLUValidator:
-    def __init__(self):
-        self.errors = []
-
-    def validate_sentence(self, sentence_lines, sent_id=None):
-        self.errors = []
-        
-        if not sentence_lines:
-            self.errors.append("Empty sentence")
-            return False
-        
-        tokens = []
-        roots = []
-        token_ids = set()
-        
-        for line_num, line in enumerate(sentence_lines, 1):
-            try:
-                fields = line.split('\t')
-                if len(fields) != 10:
-                    self.errors.append(f"Line {line_num}: Expected 10 fields, got {len(fields)}")
-                    continue
-                
-                token_id, form, lemma, upos, xpos, feats, head, deprel, deps, misc = fields
-                
-                if '-' in token_id or '.' in token_id:
-                    continue
-                
-                try:
-                    token_id_int = int(token_id)
-                    head_int = int(head)
-                except ValueError:
-                    self.errors.append(f"Line {line_num}: Invalid token ID or head")
-                    continue
-                
-                token_ids.add(token_id_int)
-                
-                if head_int == 0:
-                    roots.append(token_id_int)
-                
-                tokens.append({
-                    'id': token_id_int,
-                    'form': form,
-                    'lemma': lemma,
-                    'upos': upos,
-                    'head': head_int,
-                    'deprel': deprel
-                })
-            except Exception as e:
-                self.errors.append(f"Line {line_num}: Error: {e}")
-        
-        if len(roots) == 0:
-            self.errors.append("No root found")
-        elif len(roots) > 1:
-            self.errors.append(f"Multiple roots found: {roots}")
-        
-        for token in tokens:
-            if token['head'] != 0 and token['head'] not in token_ids:
-                self.errors.append(f"Token {token['id']} has invalid head {token['head']}")
-        
-        if not self._check_no_cycles(tokens):
-            self.errors.append("Dependency cycle detected")
-        
-        for token in tokens:
-            if not token['form'] or token['form'] == '_':
-                self.errors.append(f"Token {token['id']}: Missing form")
-            if not token['upos'] or token['upos'] == '_':
-                self.errors.append(f"Token {token['id']}: Missing UPOS")
-            if not token['deprel'] or token['deprel'] == '_':
-                self.errors.append(f"Token {token['id']}: Missing deprel")
-        
-        return len(self.errors) == 0
-
-    def _check_no_cycles(self, tokens):
-        heads = {t['id']: t['head'] for t in tokens}
-        for start in tokens:
-            visited = set()
-            current = start['id']
-            while current != 0 and current in heads:
-                if current in visited:
-                    return False
-                visited.add(current)
-                current = heads[current]
-        return True
-
-    def get_errors(self):
-        return self.errors
-
-
-# ============================================================
-# CLEANING PIPELINE (in-memory)
-# ============================================================
-def clean_sentences(sentence_blocks):
-    """
-    Take a list of CONLLU sentence blocks as strings.
-    Return: new cleaned list.
-    """
-    validator = CoNLLUValidator()
-    cleaned = []
-
-    for block in sentence_blocks:
-        lines = [l for l in block.split("\n") if not l.startswith("#")]
-        comments = [l for l in block.split("\n") if l.startswith("#")]
-        sent_id = None
-        for c in comments:
-            if c.startswith("# sent_id"):
-                sent_id = c.split("=", 1)[1].strip() if "=" in c else None
-
-        if validator.validate_sentence(lines, sent_id):
-            cleaned.append(block)
-        else:
-            print(f"[REMOVED] sent_id={sent_id}  ERRORS={validator.get_errors()}")
-
-    return cleaned
-
-
-# ============================================================
-# BUILD TRAIN SENTENCES
-# ============================================================
-train_sentences = []
-train_sentences.extend(load_from_treebank_dir(SVENSKA_PROJEKT))
-train_sentences.extend(load_from_treebank_dir(NORSKA_PROJEKT))
-train_sentences.extend(load_from_treebank_dir(DANSKA_PROJEKT))
-
-
-# ============================================================
-# HANDLE DIGPHIL MACHINE minus gold
-# ============================================================
-def map_sent_ids_by_file(directory: Path):
-    mapping = {}
-    for path in directory.glob("*.conllu"):
-        blocks = read_conllu(path)
-        ids = {extract_sent_id(b) for b in blocks if extract_sent_id(b)}
-        mapping[path.name] = ids
-    return mapping
-
-gold_ids = map_sent_ids_by_file(DIGPHIL_GOLD)
-
-for machine_file in DIGPHIL_MACHINE.glob("*.conllu"):
-    blocks = read_conllu(machine_file)
-    filename = machine_file.name
-    gold_for_this = gold_ids.get(filename, set())
-
-    for block in blocks:
-        sid = extract_sent_id(block)
-        if sid and sid in gold_for_this:
-            continue
-        train_sentences.append(block)
-
-
-# ============================================================
-# GOLD → DEV/TEST SPLIT
-# ============================================================
-gold_sentences = []
-for gold_file in DIGPHIL_GOLD.glob("*.conllu"):
-    print(f"Reading GOLD: {gold_file}")
-    gold_sentences.extend(read_conllu(gold_file))
-
-random.shuffle(gold_sentences)
-
-n = len(gold_sentences)
-dev_size = max(1, int(n * 0.10))
-
-dev_sentences = gold_sentences[:dev_size]
-test_sentences = gold_sentences[dev_size:]
-
-
-# ============================================================
-# CLEAN ALL THREE OUTPUTS
-# ============================================================
-print("Cleaning TRAIN...")
-train_sentences = clean_sentences(train_sentences)
-
-print("Cleaning DEV...")
-dev_sentences = clean_sentences(dev_sentences)
-
-print("Cleaning TEST...")
-test_sentences = clean_sentences(test_sentences)
-
-
-# ============================================================
-# WRITE FINAL OUTPUTS
-# ============================================================
-print(f"Writing TRAIN → {OUTPUT_TRAIN} ({len(train_sentences)} valid sentences)")
-write_conllu(OUTPUT_TRAIN, train_sentences)
-
-print(f"Writing DEV → {OUTPUT_DEV} ({len(dev_sentences)} valid sentences)")
-write_conllu(OUTPUT_DEV, dev_sentences)
-
-print(f"Writing TEST → {OUTPUT_TEST} ({len(test_sentences)} valid sentences)")
-write_conllu(OUTPUT_TEST, test_sentences)
-
-print("Done.")

data/depparse/sv_diachronic.dev.in.conllu

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3e187a488028eda24d6acbc32d15d75eb91d816ac0ab1e33479499ebe00c0948
-size 30651

data/depparse/sv_diachronic.test.in.conllu

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:562acbbc94d10e1dbbfb64a0678aee4c4fe9fa2d25d83ceb577fb1661ada19f3
-size 272480

data/depparse/sv_diachronic.train.in.conllu

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:10cc5ba42ab7435c03b73588eb514007d54367d696078684de08a308fb5cf989
-size 116586108

saved_models/depparse/sv_diachronic_charlm_parser.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87b032707188b5d53ed43f02af00b3d22f321b55e426400b0e8a35e2cf021bb9
-size 150761755
+oid sha256:839139656fffb5c71a8df13c7b9dc4599617ed23e79b4dd84802a7d284ebfa37
+size 150761848

saved_models/depparse/sv_diachronic_charlm_parser_checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec3577e850a19917e89cd37d0ca98f929c7b5093c05901938fe424cc53966cab
-size 450717113
+oid sha256:c6fe2ce63c2eb24d762269e19f200e59f69414020225a9221490ec7c5a191323
+size 450717215

ud/UD_Swedish-diachronic/sv_diachronic-ud-dev.conllu

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6a612aeecf56e8dabe12273071ee63ca0ed26e9ade00ba101dad2b4972b5e61c
-size 30068

ud/UD_Swedish-diachronic/sv_diachronic-ud-test.conllu

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c169b239a1f4e90357e439cc29f8c7bb764d550d88ace7de8a1bfeca28c2821d
-size 269341

ud/UD_Swedish-diachronic/sv_diachronic-ud-train.conllu

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bed616fe972a13269f92d13ff4d9f73c56bddd045ba3d5a3a47c2ff2582c7a6f
-size 108812346