Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.ipynb_checkpoints/classification_report_lr_6.0000000000e-05_test-checkpoint.csv

@@ -0,0 +1,5 @@
+Entity,Precision,Recall,F1-Score,Support
+TITLE,0.9390,0.9747,0.9565,79
+micro avg,0.9390,0.9747,0.9565,79
+macro avg,0.9390,0.9747,0.9565,79
+weighted avg,0.9390,0.9747,0.9565,79

README.md

@@ -0,0 +1,158 @@
+---
+language:
+- en
+base_model:
+- FacebookAI/xlm-roberta-large
+pipeline_tag: token-classification
+library_name: transformers
+---
+
+# Patent Title Extraction Model
+
+### Model Description
+
+**patent_titles_ner** is a fine-tuned [XLM-RoBERTa-large](https://huggingface.co/FacebookAI/xlm-roberta-large) model that has been trained on a custom dataset of OCR'd front pages of patent specifications published by the British Patent Office, and filed between 1617-1899. It has been trained to recognize the stated titles of inventions.
+
+We take the original xlm-roberta-large [weights](https://huggingface.co/FacebookAI/xlm-roberta-large/blob/main/pytorch_model.bin) and fine tune on our custom dataset for 15 epochs with a learning rate of 6e-05 and a batch size of 21. We chose the learning rate by tuning on the validation set.
+
+### Usage
+
+This model can be used with HuggingFace Transformer's Pipelines API for NER: 
+
+```python
+from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("gbpatentdata/patent_titles_ner")
+model = AutoModelForTokenClassification.from_pretrained("gbpatentdata/patent_titles_ner")
+
+
+def custom_recognizer(text, model=model, tokenizer=tokenizer, device=0):
+
+    # HF ner pipeline
+    token_level_results = pipeline("ner", model=model, device=0, tokenizer=tokenizer)(text)
+
+    # keep entities tracked
+    entities = []
+    current_entity = None
+
+    for item in token_level_results:
+
+        tag = item['entity']
+
+        # replace '▁' with space for easier reading (_ is created by the XLM-RoBERTa tokenizer)
+        word = item['word'].replace('▁', ' ')
+
+        # aggregate I-O-B tagged entities
+        if tag.startswith('B-'):
+
+            if current_entity:
+                entities.append(current_entity)
+
+            current_entity = {'type': tag[2:], 'text': word.strip(), 'start': item['start'], 'end': item['end']}
+
+        elif tag.startswith('I-'):
+
+            if current_entity and tag[2:] == current_entity['type']:
+                current_entity['text'] += word
+                current_entity['end'] = item['end']
+
+            else:
+
+                if current_entity:
+                    entities.append(current_entity)
+
+                current_entity = {'type': tag[2:], 'text': word.strip(), 'start': item['start'], 'end': item['end']}
+
+        else:
+            # deal with O tag
+            if current_entity:
+                entities.append(current_entity)
+            current_entity = None
+
+    if current_entity:
+        # add to entities
+        entities.append(current_entity)
+
+    # track entity merges
+    merged_entities = []
+
+    # merge entities of the same type
+    for entity in entities:
+        if merged_entities and merged_entities[-1]['type'] == entity['type'] and merged_entities[-1]['end'] == entity['start']:
+            merged_entities[-1]['text'] += entity['text']
+            merged_entities[-1]['end'] = entity['end']
+        else:
+            merged_entities.append(entity)
+
+    # clean up extra spaces
+    for entity in merged_entities:
+        entity['text'] = ' '.join(entity['text'].split())
+
+    # convert to list of dicts
+    return [{'class': entity['type'],
+             'entity_text': entity['text'],
+             'start': entity['start'],
+             'end': entity['end']} for entity in merged_entities]
+
+
+
+example = """
+Date of Application, 1st Aug., 1890-Accepted, 6th Sept., 1890
+COMPLETE SPECIFICATION.
+Improvements in Coin-freed Apparatus for the Sale of Goods.
+I, CHARLES LOTINGA, of 33 Cambridge Street, Lower Grange, Cardiff, in the County of Glamorgan, Gentleman,
+do hereby declare the nature of this invention and in what manner the same is to be performed,
+to be particularly described and ascertained in and by the following statement
+"""
+
+ner_results = custom_recognizer(example)
+print(ner_results)
+```
+
+### Training Data
+
+The custom dataset of front page texts of patent specifications was assembled in the following steps:
+
+1. We fine tuned a YOLO vision [model](https://huggingface.co/gbpatentdata/yolov8_patent_layouts) to detect bounding boxes around text. We use this to identify text regions on the front pages of patent specifications.
+2. We use [Google Cloud Vision](https://cloud.google.com/vision?hl=en) to OCR the detected text regions, and then concatenate the OCR text.
+3. We randomly sample 200 front page texts (and another 201 oversampled from those that contain either firm or communicant information).
+
+Our custom dataset has accurate manual labels generated by a graduate student. The final dataset is split 60-20-20 (train-val-test). In the event that the front page text is too long, we restrict the text to the first 512 tokens.
+
+### Evaluation 
+
+Our evaluation metric is F1 at the full entity-level. That is, we aggregated adjacent-indexed entities into full entities and computed F1 scores requiring an exact match. These scores for the test set are below.
+
+<table>
+  <thead>
+    <tr>
+      <th>Full Entity</th>
+      <th>Precision</th>
+      <th>Recall</th>
+      <th>F1-Score</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>TITLE</td>
+      <td>93.9%</td>
+      <td>97.5%</td>
+      <td>95.7%</td>
+    </tr>
+  </tbody>
+</table>
+
+
+## Citation 
+
+If you use our model or custom training/evaluation data in your research, please cite our accompanying paper as follows:
+
+```bibtex
+@article{bct2025,
+  title = {300 Years of British Patents},
+  author = {Enrico Berkes and Matthew Lee Chen and Matteo Tranchero},
+  journal = {arXiv preprint arXiv:2401.12345},
+  year = {2025},
+  url = {https://arxiv.org/abs/2401.12345}
+}
+```

classification_report_lr_6.0000000000e-05_test.csv

@@ -0,0 +1,5 @@
+Entity,Precision,Recall,F1-Score,Support
+TITLE,0.9390,0.9747,0.9565,79
+micro avg,0.9390,0.9747,0.9565,79
+macro avg,0.9390,0.9747,0.9565,79
+weighted avg,0.9390,0.9747,0.9565,79

classification_report_lr_6.0000000000e-05_val.csv

@@ -0,0 +1,5 @@
+Entity,Precision,Recall,F1-Score,Support
+TITLE,0.9625,0.9747,0.9686,79
+micro avg,0.9625,0.9747,0.9686,79
+macro avg,0.9625,0.9747,0.9686,79
+weighted avg,0.9625,0.9747,0.9686,79

config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "xlm-roberta-large",
+  "architectures": [
+    "XLMRobertaForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "B-TITLE",
+    "1": "I-TITLE",
+    "2": "O"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "B-TITLE": 0,
+    "I-TITLE": 1,
+    "O": 2
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c63cafaa8bd55eb4c4d6557a65a9129199e65e0d61101262f015cc21fa0c3ce
+size 2235424156

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ffb37461c391f096759f4a9bbbc329da0f36952f88bab061fcf84940c022e98
+size 17082999

tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}