Forked from talaugust/bart-sci-definition

.gitattributes

@@ -2,34 +2,26 @@
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,57 @@
+## BART Scientific Definition Generation 
+This is a finetuned BART Large model from the paper:
+
+"Generating Scientific Definitions with Controllable Complexity" 
+
+By Tal August, Katharina Reinecke, and Noah A. Smith
+
+Abstract: Unfamiliar terminology and complex language can present barriers to understanding science. Natural language processing stands to help address these issues by automatically defining unfamiliar terms. We introduce a new task and dataset for defining scientific terms and controlling the complexity of gen- erated definitions as a way of adapting to a specific reader’s background knowledge. We test four definition generation methods for this new task, finding that a sequence-to-sequence approach is most successful. We then explore the version of the task in which definitions are generated at a target complexity level. We in- troduce a novel reranking approach and find in human evaluations that it offers superior fluency while also controlling complexity, compared to several controllable generation baselines.
+
+## Description 
+
+The model is finetuned on the task of generating definitions of scientific terms. We frame our task as generating an answer to the question “What is (are) X?” Along with the question, the model takes a support document of scientific abstracted related to the term being defined. 
+
+
+## Intended use
+
+The intended use of this model is to generate definitions of scientific terms. It is NOT intended for public deployment due to the risk of hallucinated information in model output. Strong supervision of definition factuality is important for any future deployment of such a system. While hallucinated information can be damaging in any generation context, incorrect scientific definitions could mislead readers and potentially contribute to broader scientific misinformation. The model is trained on data we believe is trustworthy (e.g., questions and answers from NIH websites); however, this is no guarantee that model output will be. 
+
+## Training data
+
+The model is trained on data from two sources: Wikipedia science glossaries and a portion of the [MedQuAD dataset](https://github.com/abachaa/MedQuAD), which contains healthcare consumer questions and answers from NIH websites. For more information on these data sources, see the [github repository](https://github.com/talaugust/definition-complexity) for the paper.
+
+## How to use 
+Note that this model was trained and evaluated using transformers version 4.2.2
+
+
+      from transformers import (
+           AutoTokenizer,
+           AutoModelForSeq2SeqLM,
+           AutoConfig, 
+      )
+      
+      bart_sci_def_tokenizer = AutoTokenizer.from_pretrained("talaugust/bart-sci-definition")
+      bart_sci_def_model = AutoModelForSeq2SeqLM.from_pretrained("talaugust/bart-sci-definition")
+      
+      inputs = bart_sci_def_tokenizer("question: What is (are) surfactants? context: <P> .... <P> ...." , return_tensors='pt')
+      
+      outputs = bart_sci_def_model.generate(**inputs,
+                                       decoder_start_token_id=tokenizer.bos_token_id,
+                                       num_return_sequences=1,
+                                       num_beams=5,
+                                       max_length=64,
+                                       min_length=8,
+                                       early_stopping=True,
+                                       temperature=None,
+                                       do_sample=True,
+                                       top_k=50,
+                                       top_p=0.9,
+                                       max_input_length=1024,
+                                       no_repeat_ngram_size=3,
+                                       device=None)
+      answers = [bart_sci_def_tokenizer.decode(ans_ids, skip_special_tokens=True).strip() for ans_ids in outputs[0]]
+
+
+
+## Biases & Limitations 
+The goal of this model is to enable a wider audience of readers to understand and engage with scientific writing. A risk, though, is that such attempts might instead widen the gap to accessing scientific information. The texts in the datasets we train our models on are in General or Academic American. English. Many people, especially those who have been historically underrepresented in STEM disciplines and medicine, may not be comfortable with this dialect of English. This risks further alienating the readers we hope to serve. An important and exciting direction in NLP is making models more flexible to dialects and low-resource languages. 

config.json

@@ -0,0 +1,73 @@
+{
+  "_name_or_path": "bart_medq_wiki_gen",
+  "activation_dropout": 0.1,
+  "activation_function": "gelu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": false,
+  "architectures": [
+    "BartForConditionalGeneration"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "classif_dropout": 0.1,
+  "classifier_dropout": 0.0,
+  "d_model": 1024,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 4096,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 12,
+  "decoder_start_token_id": 2,
+  "dropout": 0.1,
+  "early_stopping": true,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 4096,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 12,
+  "eos_token_id": 2,
+  "forced_eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_position_embeddings": 1024,
+  "model_type": "bart",
+  "no_repeat_ngram_size": 3,
+  "normalize_before": false,
+  "num_beams": 4,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "scale_embedding": false,
+  "task_specific_params": {
+    "summarization": {
+      "length_penalty": 1.0,
+      "max_length": 128,
+      "min_length": 12,
+      "num_beams": 4
+    },
+    "summarization_cnn": {
+      "length_penalty": 2.0,
+      "max_length": 142,
+      "min_length": 56,
+      "num_beams": 4
+    },
+    "summarization_xsum": {
+      "length_penalty": 1.0,
+      "max_length": 62,
+      "min_length": 11,
+      "num_beams": 6
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.18.0",
+  "use_cache": true,
+  "vocab_size": 50265
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dc7d89110191442a2e31f0e85d7611f364dd00b74dd6da30c3edec19506e220
+size 1625543727