Functions to begin training, generate predictions, and remove non-context labels

ner-training/begin-training.zsh

@@ -0,0 +1,14 @@
+#!/bin/zsh
+
+source ner_venv/bin/activate
+
+train_spacy_file=$1
+dev_spacy_file=$2
+model_outdir=$3
+
+python3 -m spacy train transformer.cfg \
+--paths.train "$train_spacy_file" \
+--paths.dev "$dev_spacy_file"  \
+--gpu-id 0 \
+--output "$model_outdir"
+

ner-training/predict.py

@@ -0,0 +1,18 @@
+import spacy
+import pandas as pd
+import sys
+
+csv_path = sys.argv[1]
+custom_model_path = sys.argv[2]
+
+df = pd.read_csv(csv_path)
+texts = df['Additional Info'].to_list()
+
+trained_nlp = spacy.load(custom_model_path)
+
+for text in texts:
+    doc = trained_nlp(text)
+    print(f"TEXT: {text}")
+    print()
+    print("ENTITIES:", [(ent.text, ent.label_) for ent in doc.ents])
+    print('-'*60) 

ner-training/remove_non_context_labels.py

@@ -0,0 +1,17 @@
+import json                                                                   
+import sys
+
+file_to_filter = sys.argv[1]
+
+with open(file_to_filter, 'r') as input_file:
+    dataset = json.load(input_file)
+                                                                              
+def filter_context_labels(dataset):                                           
+    for item in dataset:                                                      
+        item['label'] = [l for l in item['label'] if 'Context' in l['labels']]
+    return dataset                                                            
+                                                                              
+filtered_data = filter_context_labels(dataset)
+
+with open('context-only-labels.json', 'w') as output_file: 
+    json.dump(filtered_data, output_file, indent=2)