Special for morphological analysis

bert_model_variant.py

@@ -0,0 +1,420 @@
+from transformers import BertConfig, BertModel
+import torch
+import re
+from torch.utils.data import DataLoader, Dataset
+from sklearn.model_selection import train_test_split, cross_validate
+import pytorch_lightning as pl
+import pandas as pd
+from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
+from torch.optim import AdamW
+from sklearn.metrics import f1_score
+
+MAX_LEN = 96
+PAD_ID = 0
+
+config = BertConfig(
+    vocab_size=40,
+    hidden_size=64,
+    num_hidden_layers=4,
+    num_attention_heads=4,
+    intermediate_size=256,
+    max_position_embeddings=MAX_LEN,
+    type_vocab_size=4
+)
+
+
+
+class MyDataset(Dataset):
+    def __init__(self, df, char2idx, label2idx, is_train=True):
+        super().__init__()
+        print(char2idx)
+        print(label2idx)
+        self.is_train = is_train
+        self.dataset = get_dataset3(df, char2idx, label2idx, is_train=is_train)
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __getitem__(self, idx):
+        return self.dataset[idx]
+
+
+    def collate_fn(self, batch):
+        collated = {
+            "input_ids": torch.IntTensor([(x[0] if self.is_train else x)["input_ids"] for x in batch]),
+            "attention_mask": torch.Tensor([(x[0] if self.is_train else x)["attention_mask"] for x in batch]),
+            "token_type_ids": torch.IntTensor([(x[0] if self.is_train else x)["token_type_ids"] for x in batch])
+        }
+        if self.is_train:
+            collated = collated, torch.IntTensor([x[1] for x in batch])
+
+        return collated
+
+
+def get_preprocessed_dfs(folder):
+    df = pd.read_csv(f"{folder}/train_data.csv").drop_duplicates()
+    df.loc[:, "Tag"] = df.Tag.apply(lambda x: "CAUS_2" if x.startswith("CAUS_") and x != "CAUS_1" else x)
+
+    cats = ['FUT_INDF_3PLF', 'FUT_INDF_NEG', 'PST_INDF_PS', 'PCP_FUT_NEG', 'PCP_FUT_DEF', 'PRES_CONT', 'PRES_2SGF', 'POSS_2SGF', 'POSS_2PLF', 'NUM_APPR3', 'NUM_APPR2', 'NUM_APPR1', 'ADVV_CONT', 'ADJECTIVE', 'PST_ITER', 'PST_INDF', 'PST_EVID', 'PRES_PST', 'POSS_3SG', 'POSS_3PL', 'POSS_2SG', 'POSS_2PL', 'POSS_1SG', 'POSS_1PL', 'NUM_COLL', 'FUT_INDF', 'ADVV_SUC', 'ADVV_NEG', 'ADVV_INT', 'ADVV_ACC', 'PST_DEF', 'NUM_ORD', 'NUMERAL', 'IMP_SGF', 'IMP_PLF', 'FUT_DEF', 'PREC_1', 'PCP_PS', 'PCP_PR', 'JUS_SG', 'JUS_PL', 'IMP_SG', 'IMP_PL', 'HOR_SG', 'HOR_PL', 'DESIDE', 'CAUS_2', 'CAUS_1', 'INF_5', 'INF_4', 'INF_3', 'INF_2', 'INF_1', 'VERB', 'REFL', 'RECP', 'PRES', 'PREM', 'PERS', 'PASS', 'COND', 'COMP', '2SGF', '2PLF', 'SUC', 'OPT', 'NOM', 'NEG', 'NEG', 'LOC', 'INT', 'GEN', 'DAT', 'ACT', 'ACC', 'ABL', '3SG', '3PL', '2SG', '2PL', '1SG', '1PL', 'SG', 'PL']
+    cats = sorted([x.lower() for x in cats], key=lambda x: (len(x), x), reverse=True)
+
+    for col in df.columns:
+        df.loc[:, col] = df[col].apply(lambda x: x.strip().lower())
+
+    def tag2list(t):
+        res = []
+        for c in cats:
+            if c in t:
+                res.append(c)
+                t = t.replace(c, "")
+        return res
+
+    df.loc[:, "Tag"] = df.Tag.apply(tag2list)
+
+    tdf = pd.read_csv(f"{folder}/test_data.csv")
+    tdf.pop("Tag")
+    for col in tdf.columns:
+        tdf.loc[:, col] = tdf[col].apply(lambda x: x.strip().lower())
+
+    return {"train": df.rename(columns={x: x.lower() for x in df.columns}), "test": tdf.rename(columns={x: x.lower() for x in tdf.columns})}
+
+def get_preprocessed_dfs2(folder):
+    df = pd.read_csv(f"{folder}/train_data.csv").drop_duplicates()
+    df.loc[:, "Tag"] = df.Tag.apply(lambda x: "CAUS_2" if x.startswith("CAUS_") and x != "CAUS_1" else x)
+
+    for col in df.columns:
+        df.loc[:, col] = df[col].apply(lambda x: x.strip().lower())
+
+    tdf = pd.read_csv(f"{folder}/test_data.csv")
+    tdf.pop("Tag")
+    for col in tdf.columns:
+        tdf.loc[:, col] = tdf[col].apply(lambda x: x.strip().lower())
+
+    return {"train": df.rename(columns={x: x.lower() for x in df.columns}), "test": tdf.rename(columns={x: x.lower() for x in tdf.columns})}
+
+def get_splits(df, test_size=0.2):
+    unique_roots = df.root.unique()
+    print("unique roots", len(unique_roots))
+    train, validation = train_test_split(unique_roots, test_size=test_size, random_state=2023)
+    print("unique train roots", len(train))
+    print("unique validation roots", len(validation))
+    train_df = df[df.root.isin(train)]
+    validation_df = df[df.root.isin(validation)]
+
+    return train_df, validation_df
+
+def get_char2idx(all_splits, special_chars=("<pad>", "<s>", "</s>")):
+    charset = set()
+    for split, df in all_splits.items():
+        charset = charset.union("".join(df.apply(lambda r: r.root + r.affix, axis=1)))
+    return {x: i for i, x in enumerate(list(special_chars) + sorted(charset))}
+
+def get_dataset(split, char2idx, label2idx, max_len=MAX_LEN, is_train=True):
+    pos2idx = {x: i for i, x in enumerate(["noun", "verb", "num", "adjective"])}
+
+    result = []
+
+    for r in split.itertuples():
+
+        input_ids = [char2idx["<s>"], pos2idx[r.pos_word], pos2idx[r.pos_root]]
+        attention_mask = [1, 1, 1]
+        token_type_ids = [0, 0, 0]
+
+        # print(r.word, r.root, r.affix)
+        for c in r.word:
+            input_ids.append(char2idx[c])
+            attention_mask.append(1)
+            token_type_ids.append(1)
+
+        for c in r.root:
+            input_ids.append(char2idx[c])
+            attention_mask.append(1)
+            token_type_ids.append(2)
+
+        for c in r.affix:
+            input_ids.append(char2idx[c])
+            attention_mask.append(1)
+            token_type_ids.append(3)
+
+        input_ids.append(char2idx["</s>"])
+        attention_mask.append(1)
+        token_type_ids.append(3)
+
+        input_ids = input_ids[:MAX_LEN]
+        attention_mask = attention_mask[:MAX_LEN]
+        token_type_ids = token_type_ids[:MAX_LEN]
+
+
+        for _ in range(MAX_LEN - len(input_ids)):
+            input_ids.append(char2idx["<pad>"])
+            attention_mask.append(0)
+            token_type_ids.append(3)
+
+        result.append(
+            {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "token_type_ids": token_type_ids,
+            }
+        )
+
+        if is_train:
+            result[-1] = (result[-1], [0 for _ in range(len(label2idx))])
+            for tag in r.tag:
+                result[-1][-1][label2idx[tag]] = 1
+            
+
+    return result
+
+def get_dataset3(split, char2idx, label2idx, max_len=MAX_LEN, is_train=True):
+    pos2idx = {x: i for i, x in enumerate(["noun", "verb", "num", "adjective"])}
+
+    result = []
+
+    for xs, r in enumerate(split.itertuples()):
+
+        input_ids = [char2idx["<s>"], pos2idx[r.pos_root]]
+        attention_mask = [1, 1]
+        token_type_ids = [0, 0]
+
+        for c in r.root:
+            input_ids.append(char2idx[c])
+            attention_mask.append(1)
+            token_type_ids.append(1)
+
+        for c in r.affix:
+            input_ids.append(char2idx[c])
+            attention_mask.append(1)
+            token_type_ids.append(2)
+
+        input_ids.append(char2idx["</s>"])
+        attention_mask.append(1)
+        token_type_ids.append(2)
+
+        input_ids = input_ids[:MAX_LEN]
+        attention_mask = attention_mask[:MAX_LEN]
+        token_type_ids = token_type_ids[:MAX_LEN]
+
+
+        for _ in range(MAX_LEN - len(input_ids)):
+            input_ids.append(char2idx["<pad>"])
+            attention_mask.append(0)
+            token_type_ids.append(2)
+
+        result.append(
+            {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "token_type_ids": token_type_ids,
+            }
+        )
+
+        if is_train:
+            result[-1] = (result[-1], label2idx[r.tag])
+
+        if xs + 1 % 1000 == 0:
+            print(input_ids)
+            print(attention_mask)
+            print(token_type_ids)
+
+    return result
+
+def get_dataset2(split, char2idx, label2idx, max_len=MAX_LEN, is_train=True):
+    pos2idx = {x: i for i, x in enumerate(["noun", "verb", "num", "adjective"])}
+
+    result = []
+
+    for xs, r in enumerate(split.itertuples()):
+
+        input_ids = [char2idx["<s>"], pos2idx[r.pos_word], pos2idx[r.pos_root]]
+        attention_mask = [1, 1, 1]
+        token_type_ids = [0, 0, 0]
+
+        # print(r.word, r.root, r.affix)
+        for c in r.word:
+            input_ids.append(char2idx[c])
+            attention_mask.append(1)
+            token_type_ids.append(1)
+
+        for c in r.root:
+            input_ids.append(char2idx[c])
+            attention_mask.append(1)
+            token_type_ids.append(2)
+
+        for c in r.affix:
+            input_ids.append(char2idx[c])
+            attention_mask.append(1)
+            token_type_ids.append(3)
+
+        input_ids.append(char2idx["</s>"])
+        attention_mask.append(1)
+        token_type_ids.append(3)
+
+        input_ids = input_ids[:MAX_LEN]
+        attention_mask = attention_mask[:MAX_LEN]
+        token_type_ids = token_type_ids[:MAX_LEN]
+
+
+        for _ in range(MAX_LEN - len(input_ids)):
+            input_ids.append(char2idx["<pad>"])
+            attention_mask.append(0)
+            token_type_ids.append(3)
+
+        result.append(
+            {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "token_type_ids": token_type_ids,
+            }
+        )
+
+        if is_train:
+            result[-1] = (result[-1], label2idx[r.tag])
+
+        if xs + 1 % 10000 == 0:
+            print(input_ids)
+            print(attention_mask)
+            print(token_type_ids)
+
+
+    return result
+
+def train_model(epochs=100, batch_size=400, data_folder="../Downloads/"):
+    dfs = get_preprocessed_dfs2(data_folder)
+    train, val = get_splits(dfs["train"])
+    char2idx = get_char2idx(dfs)
+    # label2idx = {j: i for i, j in enumerate(sorted(set([x for y in dfs["train"].tag for x in y])))}
+    label2idx = {l: i for i, l in enumerate(dfs["train"].tag.unique())}
+
+    model = MyModel2(config, label2idx, char2idx, 0.5)
+    checkpoint_callback = ModelCheckpoint(
+        dirpath="fmicro_weights",
+        save_top_k=3,
+        monitor="fmicro",
+        mode="max",
+        filename="{epoch}-{step}",
+    )
+    trainer = pl.Trainer(
+        deterministic=True,
+        max_epochs=epochs,
+        callbacks=[EarlyStopping(monitor="fmicro", mode="max"), checkpoint_callback],
+        log_every_n_steps=30,
+    )
+
+    train_dataset = MyDataset(train, char2idx, label2idx)
+    validation_dataset = MyDataset(val, char2idx, label2idx)
+    trainer.fit(model, DataLoader(train_dataset, batch_size=400, collate_fn=train_dataset.collate_fn), DataLoader(validation_dataset, batch_size=400, collate_fn=validation_dataset.collate_fn))
+
+    best_model_path = [c for c in trainer.callbacks if isinstance(c, ModelCheckpoint)][0].best_model_path
+
+    model.load_state_dict(torch.load(best_model_path)["state_dict"])
+
+    return model, train, val, dfs["test"]
+
+
+class MyModel(pl.LightningModule):
+    def __init__(self, config, label2idx, threshold, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.threshold = threshold
+        self.char2idx = char2idx
+        self.label2idx = label2idx
+        self.idx2label = {i: l for l, i in label2idx.items()}
+        self.bert = BertModel(config)
+        self.dropout = torch.nn.Dropout(0.3)
+        self.proj = torch.nn.Linear(config.hidden_size, len(label2idx))
+
+
+    def common_step(self, batch):
+        X, _ = batch
+        hidden = self.bert(**X)[1]
+        return self.proj(self.dropout(hidden))
+
+    def training_step(self, batch, batch_idx):
+        # print(batch)
+        logits = self.common_step(batch)
+        loss = torch.nn.BCEWithLogitsLoss()(logits, batch[1].float())
+        self.log("train_loss", loss.mean(), on_step=True, on_epoch=True, prog_bar=True)
+
+        return loss
+
+    def validation_step(self, batch, batch_idx):
+        # print(batch[0]["input_ids"])
+        # print(batch[0]["token_type_ids"])
+        logits = self.common_step(batch)
+        # print(logits)
+        # print(batch[1])
+        loss = torch.nn.BCEWithLogitsLoss()(logits, batch[1].float())
+        self.log("loss", loss.mean(), prog_bar=True, on_epoch=True)
+
+        return logits, loss
+
+    def test_step(self, batch, batch_idx):
+        return self.common_step((batch, []))
+
+    def forward(self, batch, batch_idx):
+        return self.common_step((batch, []))
+    
+    def configure_optimizers(self):
+        return AdamW(params=self.parameters())
+
+class MyModel2(pl.LightningModule):
+    def __init__(self, config, label2idx, char2idx, threshold, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.threshold = threshold
+        self.char2idx = char2idx
+        self.fscore = 0.0
+        self.label2idx = label2idx
+        self.idx2label = {i: l for l, i in label2idx.items()}
+        self.bert = BertModel(config)
+        self.dropout = torch.nn.Dropout(0.3)
+        self.proj = torch.nn.Linear(config.hidden_size, len(label2idx))
+
+
+    def common_step(self, batch):
+        X, _ = batch
+        hidden = self.bert(**X)[1]
+        return self.proj(self.dropout(hidden))
+
+    def training_step(self, batch, batch_idx):
+        # print(batch)
+        logits = self.common_step(batch)
+        loss = torch.nn.CrossEntropyLoss()(logits.view(-1, len(self.label2idx)), batch[1].view(-1).long())
+        self.log("train_loss", loss.mean(), on_step=True, on_epoch=True, prog_bar=True)
+
+        return loss
+
+    def validation_step(self, batch, batch_idx):
+        # print(batch[0]["input_ids"])
+        # print(batch[0]["token_type_ids"])
+        logits = self.common_step(batch)
+        # print(logits)
+        # print(batch[1])
+        loss = torch.nn.CrossEntropyLoss()(logits.view(-1, len(self.label2idx)), batch[1].view(-1).long())
+        for p in logits:
+            self.predos.append(self.idx2label[p.argmax().cpu().item()])
+        for t in batch[1]:
+            self.trues.append(self.idx2label[t.cpu().item()])
+        self.log("loss", loss.mean(), prog_bar=True, on_epoch=True)
+        self.log("fmicro", self.fscore, prog_bar=True, on_epoch=True)
+
+        return logits, loss
+
+    def on_validation_start(self):
+        self.predos = []
+        self.trues = []
+
+    def on_validation_end(self):
+        self.fscore = f1_score(self.trues, self.predos, average="micro")
+
+    def test_step(self, batch, batch_idx):
+        return self.common_step((batch, []))
+
+    def forward(self, batch, batch_idx):
+        return self.common_step((batch, []))
+    
+    def configure_optimizers(self):
+        return AdamW(params=self.parameters())
+
+    def predict(self, dataloader):
+        pass
+

dev.ipynb

@@ -0,0 +1,182 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import random\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.ensemble import RandomForestClassifier\n",
+    "from sklearn.metrics import f1_score\n",
+    "from sklearn.preprocessing import LabelEncoder"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SEED = 1\n",
+    "random.seed(SEED)\n",
+    "np.random.seed(SEED)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train = pd.read_csv('train_lr.csv').sort_values(by=['PoS_word', 'Tag', 'Affix'])\n",
+    "test = pd.read_csv('test_lr.csv').sort_values(by=['PoS_word', 'Tag', 'Affix'])\n",
+    "df = pd.concat([train, test], ignore_index=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X = df[['Word', 'Root', 'Affix', 'PoS_root', 'PoS_word']]\n",
+    "y = df['Tag']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_pr = pd.get_dummies(X)\n",
+    "le = LabelEncoder()\n",
+    "y = le.fit_transform(y)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_X = X_pr.iloc[:train.shape[0]]\n",
+    "train_y = y[:train.shape[0]]\n",
+    "train_X, val_X, train_y, val_y = train_test_split(train_X, train_y, test_size=0.05, random_state=SEED)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style>#sk-container-id-4 {color: black;background-color: white;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestClassifier(random_state=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" checked><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(random_state=1)</pre></div></div></div></div></div>"
+      ],
+      "text/plain": [
+       "RandomForestClassifier(random_state=1)"
+      ]
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rf = RandomForestClassifier(n_estimators=100, random_state=SEED)\n",
+    "rf.fit(train_X, train_y)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rf_predict_result = rf.predict(val_X)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "F1 score: 0.9099025974025974\n"
+     ]
+    }
+   ],
+   "source": [
+    "f1_micro = f1_score(val_y, rf_predict_result, average='micro')\n",
+    "print(\"F1 score:\", f1_micro)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_X = X_pr.iloc[train.shape[0]:]\n",
+    "predictions = rf.predict(test_X)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test['Tag'] = le.inverse_transform(predictions)\n",
+    "test[['Word', 'Root', 'Affix', 'Tag']].to_csv('my_submission2.csv', index=False, header=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "  "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "myenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

dev.py

@@ -0,0 +1,40 @@
+import random
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import f1_score
+from sklearn.preprocessing import LabelEncoder
+
+SEED = 1
+random.seed(SEED)
+np.random.seed(SEED)
+
+train = pd.read_csv('train_lr.csv').sort_values(by=['PoS_word', 'Tag', 'Affix'])
+test = pd.read_csv('test_lr.csv').sort_values(by=['PoS_word', 'Tag', 'Affix'])
+df = pd.concat([train, test], ignore_index=True)
+
+X = df[['Word', 'Root', 'Affix', 'PoS_root', 'PoS_word']]
+y = df['Tag']
+
+X_pr = pd.get_dummies(X)
+le = LabelEncoder()
+y = le.fit_transform(y)
+
+train_X = X_pr.iloc[:train.shape[0]]
+train_y = y[:train.shape[0]]
+train_X, val_X, train_y, val_y = train_test_split(train_X, train_y, test_size=0.05, random_state=SEED)
+
+rf = RandomForestClassifier(n_estimators=100, random_state=SEED)
+rf.fit(train_X, train_y)
+
+rf_predict_result = rf.predict(val_X)
+
+f1_micro = f1_score(val_y, rf_predict_result, average='micro')
+print("F1 score:", f1_micro)
+
+test_X = X_pr.iloc[train.shape[0]:]
+predictions = rf.predict(test_X)
+
+test['Tag'] = le.inverse_transform(predictions)
+test[['Word', 'Root', 'Affix', 'Tag']].to_csv('my_submission2.csv', index=False, header=True)

train.py

@@ -0,0 +1,205 @@
+# %%
+import pandas as pd
+from sklearn.model_selection import train_test_split, GridSearchCV
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score, f1_score
+import joblib
+from scipy.sparse import hstack
+
+
+# Read the data from the CSV file
+
+from collections import defaultdict
+
+def split_train_left_right(data):
+    sorted = data.sort_values(['Tag', 'Affix'])
+    sorted = sorted.drop_duplicates(subset=['Word', 'Tag'])
+
+    tags = defaultdict(list)
+
+    left = []
+    right = []
+
+    for i, row in sorted.iterrows():
+        # word = f"{row['Word']}{row['Affix']}"
+        word = row['Word']
+        tag = row['Tag']
+        
+        if tags[word] and (tag not in tags[word]):
+            # print(tag not in tags['word'])
+            left.append(row)
+        else:
+            right.append(row)
+
+        tags[word].append(tag)
+
+    right_df = pd.DataFrame(right)
+    left_df = pd.DataFrame(left)
+
+    return right_df, left_df
+
+filepath = "train_fixed.csv"
+data = pd.read_csv(filepath)
+
+right_df, left_df = split_train_left_right(data)
+# right_df = pd.read_csv('right.csv')
+# left_df = pd.read_csv('left.csv')
+
+
+# %%
+for (side, df) in [('right', right_df), ('left', left_df)]:
+    # Get unique categories from "PoS_word" column
+    categories = df["PoS_word"].unique()
+
+    category_res = {}
+
+    for category in categories:
+        print(f"Category: {category}")
+
+        # Filter data for the current category
+        category_data = df[df["PoS_word"] == category]
+        print(category_data.shape)
+
+        category_data['text_length'] = category_data['Affix'].apply(lambda x: len(x))
+        category_data['word_length'] = category_data['Word'].apply(lambda x: len(x))
+        category_data['ү_count'] = category_data['Word'].apply(lambda x: x.count('ү'))
+        category_data['ө_count'] = category_data['Word'].apply(lambda x: x.count('ө'))
+
+        # Splitting data into train and test
+        X = category_data["Affix"]
+        y = category_data["Tag"]
+        # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+        # Feature extraction
+        vectorizer = TfidfVectorizer(analyzer="char", ngram_range=(1, 5))
+        X_train_tfidf = vectorizer.fit_transform(X)
+        # print(len(vectorizer.vocabulary_))
+
+        X_train_combined = hstack([X_train_tfidf, category_data[['text_length', 'ү_count', 'ө_count']]])
+        # X_test_combined = hstack([X_test_tfidf, category_data[['text_length', 'ү_count', 'ө_count']]])
+
+        # X_test_vec = vectorizer.transform(X)
+
+        model = RandomForestClassifier(n_estimators=300)
+        model.fit(X_train_combined, y)
+
+        # Save the best model for the category
+        # category_models[category] = (model, vectorizer)
+
+        # Predict on the test data using the best model
+        y_pred = model.predict(X_train_combined)
+
+        # res_df = pd.DataFrame()
+        # res_df['pred'] = y_pred
+        # res_df['orig'] = y
+        category_data['pred'] = y_pred
+        category_res[category] = category_data
+
+
+        # Calculate accuracy and F1 score   
+        accuracy = accuracy_score(y, y_pred)
+        f1 = f1_score(y, y_pred, average="weighted")
+
+
+
+        print("Accuracy:", accuracy)
+        print("F1 Score:", f1)
+        print(model)
+
+    # Save the models and vectorizers
+    # for category, (model, vectorizer) in category_models.items():
+        model_filepath = f"artefacts/model_{category}_{side}.joblib"
+        vectorizer_filepath = f"artefacts/vectorizer_{category}_{side}.joblib"
+        joblib.dump(model, model_filepath)
+        joblib.dump(vectorizer, vectorizer_filepath)
+
+# %%
+filepath = "test_fixed.csv"
+data = pd.read_csv(filepath)
+
+
+def split_test_left_right(data):
+    sorted = data.sort_values(['Affix'])
+    # sorted = sorted.drop_duplicates(subset=['Word', 'Tag'])
+
+    tags = defaultdict(list)
+
+    left = []
+    right = []
+
+    for i, row in sorted.iterrows():
+        word = row['Word']        
+
+        if tags[word]:
+            # print(tag not in tags['word'])
+            left.append(row)
+        else:
+            right.append(row)
+        tags[word].append(word)
+
+
+    right_df = pd.DataFrame(right)
+    left_df = pd.DataFrame(left)
+
+    return right_df, left_df
+
+right_df, left_df = split_test_left_right(data)
+# right_df = pd.read_csv('right.csv')
+# left_df = pd.read_csv('left.csv')
+# left_df[left_df['Word'] == 'божомолдчу']
+
+# %%
+result_dfs = []
+for (side, df) in [('right', right_df), ('left', left_df)]:
+    # Get unique categories from "PoS_word" column
+    print(side)
+    categories = df["PoS_word"].unique()
+
+    # category_models = {}
+
+    for category in categories:
+        print(f"Category: {category}, side: {side}")
+
+        # Filter data for the current category
+        category_data = df[df["PoS_word"] == category]
+        print(category_data.shape)
+
+
+        category_data['text_length'] = category_data['Affix'].apply(lambda x: len(x))
+        category_data['word_length'] = category_data['Word'].apply(lambda x: len(x))
+        category_data['ү_count'] = category_data['Word'].apply(lambda x: x.count('ү'))
+        category_data['ө_count'] = category_data['Word'].apply(lambda x: x.count('ө'))
+
+
+        # Splitting data into train and test
+        X = category_data["Affix"]
+        y = category_data["Tag"]
+        # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+
+
+        # Feature extraction
+        vectorizer =  joblib.load(f"artefacts/vectorizer_{category}_{side}.joblib")
+        X_train_tfidf = vectorizer.transform(X)
+
+
+        # X_test_vec = vectorizer.transform(X)
+
+        model = joblib.load(f"artefacts/model_{category}_{side}.joblib")
+
+        # Save the best model for the category
+        # category_models[category] = (model, vectorizer)
+
+        X_train_combined = hstack([X_train_tfidf, category_data[['text_length', 'ү_count', 'ө_count']]])
+        # X
+        # Predict on the test data using the best model
+        y_pred = model.predict(X_train_combined)
+
+        category_data['Tag'] = y_pred
+        result_dfs.append(category_data)
+# %%
+
+pd.concat(result_dfs).to_csv('file_pred_12.csv', index=False)
+
+# %%