bootleg_uncased, entity_db, nq

.gitattributes

@@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+entity_db/entity_mappings/alias2id/vocabulary_trie.marisa filter=lfs diff=lfs merge=lfs -text
+entity_db/entity_mappings/alias2qids_unfiltered.json filter=lfs diff=lfs merge=lfs -text
+entity_db/entity_mappings/alias2qids/record_trie.marisa filter=lfs diff=lfs merge=lfs -text
+entity_db/entity_mappings/alias2qids/vocabulary_trie.marisa filter=lfs diff=lfs merge=lfs -text
+entity_db/entity_mappings/qid2desc.json filter=lfs diff=lfs merge=lfs -text
+entity_db/entity_mappings/qid2eid/vocabulary_trie.marisa filter=lfs diff=lfs merge=lfs -text
+entity_db/entity_mappings/qid2title.json filter=lfs diff=lfs merge=lfs -text
+entity_db/kg_mappings/qid2relations.json filter=lfs diff=lfs merge=lfs -text
+entity_db/kg_mappings/qid2relations/record_trie.marisa filter=lfs diff=lfs merge=lfs -text
+entity_db/kg_mappings/qid2relations/value_vocabulary_trie.marisa filter=lfs diff=lfs merge=lfs -text
+entity_db/type_mappings/wiki/qid2typenames/record_trie.marisa filter=lfs diff=lfs merge=lfs -text
+entity_db/type_mappings/wiki/qid2typenames/vocabulary_trie.marisa filter=lfs diff=lfs merge=lfs -text

bootleg_uncased/bootleg_config.yaml

@@ -0,0 +1,72 @@
+data_config:
+  context_mask_perc: 0.0
+  data_dir: /home/data/bootleg-data/wiki_title_0122
+  data_prep_dir: prep
+  dev_dataset:
+    file: merged_sample.jsonl
+    use_weak_label: true
+  entity_dir: /home/data/bootleg-data/wiki_title_0122/entity_db
+  entity_kg_data:
+    kg_symbols_dir: kg_mappings
+    use_entity_kg: true
+  entity_type_data:
+    type_symbols_dir: type_mappings/wiki
+    use_entity_types: true
+  eval_slices:
+  - unif_all
+  - unif_NS_all
+  - unif_HD
+  - unif_TO
+  - unif_TL
+  - unif_TS
+  max_ent_len: 128
+  max_seq_len: 128
+  max_seq_window_len: 64
+  overwrite_preprocessed_data: false
+  test_dataset:
+    file: merged_sample.jsonl
+    use_weak_label: true
+  train_dataset:
+    file: train.jsonl
+    use_weak_label: true
+  train_in_candidates: true
+  use_entity_desc: true
+  word_embedding:
+    bert_model: bert-base-uncased
+    cache_dir: bootleg-data/pretrained_bert_models
+    context_layers: 6
+    entity_layers: 6
+emmental:
+  checkpoint_all: true
+  checkpoint_freq: 1
+  checkpoint_metric: NED/Bootleg/dev/final_loss/acc_boot:max
+  checkpointing: true
+  clear_intermediate_checkpoints: false
+  counter_unit: batch
+  evaluation_freq: 21432
+  fp16: true
+  grad_clip: 1.0
+  gradient_accumulation_steps: 1
+  l2: 0.01
+  log_path: bootleg-logs/bootleg_wiki
+  lr: 2e-5
+  lr_scheduler: linear
+  n_steps: 428648
+  online_eval: false
+  dataparallel: false
+  use_exact_log_path: true
+  warmup_percentage: 0.1
+  write_loss_per_step: true
+  writer: json
+model_config:
+  hidden_size: 200
+  normalize: true
+  temperature: 0.01
+run_config:
+  dataloader_threads: 2
+  dataset_threads: 20
+  eval_batch_size: 32
+  log_level: DEBUG
+  spawn_method: forkserver
+train_config:
+  batch_size: 32

bootleg_uncased/bootleg_wiki.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:167fac90c0229f435a934f07b8e6454c6ef8c360b60776b2c18360d67c0f2928
+size 536996441

entity_db/entity_mappings/alias2id/config.json

@@ -0,0 +1 @@
+{"max_id":15202496}

entity_db/entity_mappings/alias2id/itoexti.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0094aab9f7ae34b6a0ef9ee3588024e9acc8c8a41e0a5246aa8728b86c522cb5
+size 121620104

entity_db/entity_mappings/alias2id/vocabulary_trie.marisa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1044277778512c550993ae3931d1a98fd1046facfd1f1d0309ec45dd87c543df
+size 74994488

entity_db/entity_mappings/alias2qids/max_value.json

@@ -0,0 +1 @@
+30

entity_db/entity_mappings/alias2qids/record_trie.marisa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:736f40faa2d7b2a498461fa0453ecd961acdd01636a1b19ee50624d8226d3738
+size 171802008

entity_db/entity_mappings/alias2qids/vocabulary_trie.marisa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37ccf342373cab96c74d29f6aeaefbb0ad837b44a10f4f3f1517ea397b5b9097
+size 10454104

entity_db/entity_mappings/alias2qids_unfiltered.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:289eb4c51ee81953ad43661c9094daefcc9d58a27ceab9e616535c81b24078ac
+size 656136049

entity_db/entity_mappings/config.json

@@ -0,0 +1 @@
+{"max_candidates":30,"datetime":"2021-10-29 00:00:29.790133"}

entity_db/entity_mappings/qid2desc.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693f838d73bcfb20b0744d60a7f51b982b0420f00725343d4ab0f89f6583f990
+size 799494517

entity_db/entity_mappings/qid2eid/config.json

@@ -0,0 +1 @@
+{"max_id":5832699}

entity_db/entity_mappings/qid2eid/itoexti.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27b7573f1c71aa9abd828ea8f3d643ed5fee9c02a37f9d0e43f31a75d9a8cd0
+size 46661720

entity_db/entity_mappings/qid2eid/vocabulary_trie.marisa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37ccf342373cab96c74d29f6aeaefbb0ad837b44a10f4f3f1517ea397b5b9097
+size 10454104

entity_db/entity_mappings/qid2title.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83b87dcbc4e67d1c6db11a2d772465915e79c05b67f4cd151fa17973782f7b41
+size 203209217

entity_db/kg_mappings/config.json

@@ -0,0 +1 @@
+{"max_connections":25}

entity_db/kg_mappings/qid2relations.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fc5a950f6d925e5772474f6b6b274962da0a51be79990f39029d670d03f39e0
+size 1067890225

entity_db/kg_mappings/qid2relations/max_value.json

@@ -0,0 +1 @@
+50

entity_db/kg_mappings/qid2relations/record_trie.marisa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2f58f5b9331562dc1050d31eaaa313a4c79d3e97856652c9dd769662f4fe9c5
+size 285670048

entity_db/kg_mappings/qid2relations/value_vocabulary_trie.marisa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d8559e87ef56a78e777742b008e6d5e2e6946a5b8ecc50daf3c80489dcfcd9a
+size 7352208

entity_db/type_mappings/wiki/config.json

@@ -0,0 +1 @@
+{"max_types":20}

entity_db/type_mappings/wiki/qid2typenames/max_value.json

@@ -0,0 +1 @@
+20

entity_db/type_mappings/wiki/qid2typenames/record_trie.marisa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e2d5477701a18fd5edc63c6054e59853a7dcb359bcc859c7bdff9670ac76a4f
+size 22718048

entity_db/type_mappings/wiki/qid2typenames/vocabulary_trie.marisa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf8e4bd7b0f11b868fdb2605c7fefed819ff1f3b79c8bb844a7fda63dbd013b1
+size 160824

nq/test_50.jsonl

@@ -0,0 +1,50 @@
+{"sentence": "Who did the voice of the magician in Frosty the Snowman", "sent_idx_unq": 0, "aliases": ["frosty the snowman"], "spans": [[8, 11]], "qids": ["Q5506238"], "gold": [true]}
+{"sentence": "What is considered the Outer Banks in North Carolina", "sent_idx_unq": 1, "aliases": ["outer banks", "north carolina"], "spans": [[4, 6], [7, 9]], "qids": ["Q1517373", "Q1454"], "gold": [true, true]}
+{"sentence": "The Nashville sound brought a polished and cosmopolitan sound to country music by", "sent_idx_unq": 2, "aliases": ["nashville sound", "country music"], "spans": [[1, 3], [10, 12]], "qids": ["Q1751782", "Q83440"], "gold": [true, true]}
+{"sentence": "What channel is the Premier League on in France", "sent_idx_unq": 3, "aliases": ["premier league", "france"], "spans": [[4, 6], [8, 9]], "qids": ["Q9448", "Q142"], "gold": [true, true]}
+{"sentence": "I Love It ( feat . Charli XCX ) Icona Pop", "sent_idx_unq": 4, "aliases": ["i love it", "charli xcx", "icona pop"], "spans": [[0, 3], [6, 8], [9, 11]], "qids": ["Q3273659", "Q5084390", "Q808703"], "gold": [true, true, true]}
+{"sentence": "The U.S. Supreme Court hears appeals from circuit courts", "sent_idx_unq": 5, "aliases": ["u.s. supreme court", "circuit courts"], "spans": [[1, 4], [7, 9]], "qids": ["Q11201", "Q1641851"], "gold": [true, true]}
+{"sentence": "Why does the author say that the vampire in Nosferatu is named Count Orlok and not Count Dracula", "sent_idx_unq": 6, "aliases": ["nosferatu", "count orlok", "count dracula"], "spans": [[9, 10], [12, 14], [16, 18]], "qids": ["Q151895", "Q1442062", "Q3266236"], "gold": [true, true, true]}
+{"sentence": "Is there an active volcano in New Zealand", "sent_idx_unq": 7, "aliases": ["new zealand"], "spans": [[6, 8]], "qids": ["Q664"], "gold": [true]}
+{"sentence": "Once Upon a Time Season 6 episode list", "sent_idx_unq": 8, "aliases": ["once upon a time season 6"], "spans": [[0, 6]], "qids": ["Q23301616"], "gold": [true]}
+{"sentence": "Who is the former co-chairman Goldman Sachs who became a U.S. Secretary of the Treasury", "sent_idx_unq": 9, "aliases": ["goldman sachs", "us secretary of the treasury"], "spans": [[5, 7], [10, 15]], "qids": ["Q193326", "Q4215834"], "gold": [true, true]}
+{"sentence": "Who plays Norman Bates in the TV show", "sent_idx_unq": 10, "aliases": ["norman bates"], "spans": [[2, 4]], "qids": ["Q2467893"], "gold": [true]}
+{"sentence": "Hitchhiker 's Guide to the Galaxy Slartibartfast quotes", "sent_idx_unq": 11, "aliases": ["hitchhiker 's guide to the galaxy", "slartibartfast"], "spans": [[0, 6], [6, 7]], "qids": ["Q25169", "Q779920"], "gold": [true, true]}
+{"sentence": "What was Dennis Hopper 's bike in Easy Rider", "sent_idx_unq": 12, "aliases": ["dennis hopper", "easy rider"], "spans": [[2, 4], [7, 9]], "qids": ["Q102711", "Q503638"], "gold": [true, true]}
+{"sentence": "Where was 10 Things I Hate About You filmed school", "sent_idx_unq": 13, "aliases": ["10 things i hate about you"], "spans": [[2, 8]], "qids": ["Q169082"], "gold": [true]}
+{"sentence": "Where does the last name Aponte come from", "sent_idx_unq": 14, "aliases": ["aponte"], "spans": [[5, 6]], "qids": ["Q21502338"], "gold": [true]}
+{"sentence": "What teams are in the FA Cup Final", "sent_idx_unq": 15, "aliases": ["fa cup final"], "spans": [[5, 8]], "qids": ["Q4484477"], "gold": [true]}
+{"sentence": "Where did Britain create colonies for its empire", "sent_idx_unq": 16, "aliases": ["britain", "empire"], "spans": [[2, 3], [7, 8]], "qids": ["Q161885", "Q8680"], "gold": [true, true]}
+{"sentence": "Is it a bank holiday today in Spain", "sent_idx_unq": 17, "aliases": ["bank holiday", "spain"], "spans": [[3, 5], [7, 8]], "qids": ["Q277436", "Q29"], "gold": [true, true]}
+{"sentence": "1970 World Cup semi final Italy vs Germany", "sent_idx_unq": 18, "aliases": ["1970 world cup", "italy", "germany"], "spans": [[0, 3], [5, 6], [7, 8]], "qids": ["Q132664", "Q676899", "Q43310"], "gold": [true, true, true]}
+{"sentence": "Who played the bank robber in Dirty Harry", "sent_idx_unq": 19, "aliases": ["dirty harry"], "spans": [[6, 8]], "qids": ["Q110206"], "gold": [true]}
+{"sentence": "What is the worth of the Catholic Church", "sent_idx_unq": 20, "aliases": ["catholic church"], "spans": [[6, 8]], "qids": ["Q9592"], "gold": [true]}
+{"sentence": "The pair of hand drums used in Indian classical music is called", "sent_idx_unq": 21, "aliases": ["indian classical music"], "spans": [[7, 10]], "qids": ["Q1323698"], "gold": [true]}
+{"sentence": "Game of Thrones Season 1 white hair girl", "sent_idx_unq": 22, "aliases": ["game of thrones season 1"], "spans": [[0, 5]], "qids": ["Q1658029"], "gold": [true]}
+{"sentence": "Where is Israel located on the world map", "sent_idx_unq": 23, "aliases": ["israel", "world map"], "spans": [[2, 3], [6, 8]], "qids": ["Q801", "Q653848"], "gold": [true, true]}
+{"sentence": "Who played Smiley in Tinker Tailor Soldier Spy", "sent_idx_unq": 24, "aliases": ["tinker tailor soldier spy"], "spans": [[4, 8]], "qids": ["Q681962"], "gold": [true]}
+{"sentence": "Which of these was not an export of Ancient Greece", "sent_idx_unq": 25, "aliases": ["ancient greece"], "spans": [[8, 10]], "qids": ["Q11772"], "gold": [true]}
+{"sentence": "Who opened and closed the 1960 Winter Olympics", "sent_idx_unq": 26, "aliases": ["1960 winter olympics"], "spans": [[5, 8]], "qids": ["Q9634"], "gold": [true]}
+{"sentence": "I see the river Tiber foaming with much blood", "sent_idx_unq": 27, "aliases": ["river tiber"], "spans": [[3, 5]], "qids": ["Q13712"], "gold": [true]}
+{"sentence": "What causes a dead zone in the ocean", "sent_idx_unq": 28, "aliases": ["dead zone"], "spans": [[3, 5]], "qids": ["Q1236263"], "gold": [true]}
+{"sentence": "Who plays Claire Underwood 's mom on House of Cards", "sent_idx_unq": 29, "aliases": ["claire underwood", "house of cards"], "spans": [[2, 4], [7, 10]], "qids": ["Q14915624", "Q3330940"], "gold": [true, true]}
+{"sentence": "What is the T Rex name in Land Before Time", "sent_idx_unq": 30, "aliases": ["t rex", "land before time"], "spans": [[3, 5], [7, 10]], "qids": ["Q14332", "Q192403"], "gold": [true, true]}
+{"sentence": "Who does Oregon state play in the College World Series", "sent_idx_unq": 31, "aliases": ["oregon state", "college world series"], "spans": [[2, 4], [7, 10]], "qids": ["Q7101349", "Q787505"], "gold": [true, true]}
+{"sentence": "Who played in the last 3 NBA Finals", "sent_idx_unq": 32, "aliases": ["nba finals"], "spans": [[6, 8]], "qids": ["Q842375"], "gold": [true]}
+{"sentence": "UK national debt as percentage of GDP by year", "sent_idx_unq": 33, "aliases": ["uk national debt", "gdp"], "spans": [[0, 3], [6, 7]], "qids": ["Q611713", "Q12638"], "gold": [true, true]}
+{"sentence": "Who 's doing the halftime show in 2018", "sent_idx_unq": 34, "aliases": ["halftime show"], "spans": [[4, 6]], "qids": ["Q902899"], "gold": [true]}
+{"sentence": "Reasons why South Africa should include renewable energy in its energy mix", "sent_idx_unq": 35, "aliases": ["south africa", "renewable energy"], "spans": [[2, 4], [6, 8]], "qids": ["Q258", "Q12705"], "gold": [true, true]}
+{"sentence": "When was the first freeway built in Los Angeles", "sent_idx_unq": 36, "aliases": ["los angeles"], "spans": [[7, 9]], "qids": ["Q65"], "gold": [true]}
+{"sentence": "Landmark Supreme Court cases dealing with the First Amendment", "sent_idx_unq": 37, "aliases": ["supreme court", "first amendment"], "spans": [[1, 3], [7, 9]], "qids": ["Q11201", "Q12616"], "gold": [true, true]}
+{"sentence": "When was the Wizard of Oz made in Technicolor", "sent_idx_unq": 38, "aliases": ["wizard of oz", "technicolor"], "spans": [[3, 6], [8, 9]], "qids": ["Q193695", "Q674564"], "gold": [true, true]}
+{"sentence": "Where is the movie Call Me by Your Name filmed", "sent_idx_unq": 39, "aliases": ["call me by your name"], "spans": [[4, 9]], "qids": ["Q25136757"], "gold": [true]}
+{"sentence": "Where does the last name Vigil come from", "sent_idx_unq": 40, "aliases": ["vigil"], "spans": [[5, 6]], "qids": ["Q16878937"], "gold": [true]}
+{"sentence": "Who was President of the United States in 1938", "sent_idx_unq": 41, "aliases": ["president of the united states"], "spans": [[2, 7]], "qids": ["Q11696"], "gold": [true]}
+{"sentence": "Who proposed the coordinate system to describe the position of a point in a plane accurately", "sent_idx_unq": 42, "aliases": ["coordinate system"], "spans": [[3, 5]], "qids": ["Q62912"], "gold": [true]}
+{"sentence": "When was last time England were in a World Cup semi final", "sent_idx_unq": 43, "aliases": ["england", "world cup"], "spans": [[4, 5], [8, 10]], "qids": ["Q47762", "Q19317"], "gold": [true, true]}
+{"sentence": "The representative of the British crown in NZ", "sent_idx_unq": 44, "aliases": ["british crown", "nz"], "spans": [[4, 6], [7, 8]], "qids": ["Q21941952", "Q664"], "gold": [true, true]}
+{"sentence": "When did Rangers last win the Scottish Cup", "sent_idx_unq": 45, "aliases": ["rangers", "scottish cup"], "spans": [[2, 3], [6, 8]], "qids": ["Q19597", "Q308822"], "gold": [true, true]}
+{"sentence": "Who controls the past controls the future Rage Against the Machine", "sent_idx_unq": 46, "aliases": ["rage against the machine"], "spans": [[7, 11]], "qids": ["Q72092"], "gold": [true]}
+{"sentence": "Who is Mariah Carey talking about in We Belong Together", "sent_idx_unq": 47, "aliases": ["mariah carey", "we belong together"], "spans": [[2, 4], [7, 10]], "qids": ["Q41076", "Q129158"], "gold": [true, true]}
+{"sentence": "What was the Japanese motivation for bombing Pearl Harbor", "sent_idx_unq": 48, "aliases": ["japanese", "pearl harbor"], "spans": [[3, 4], [7, 9]], "qids": ["Q188712", "Q127091"], "gold": [true, true]}
+{"sentence": "Cast of characters in Fiddler on the Roof", "sent_idx_unq": 49, "aliases": ["fiddler on the roof"], "spans": [[4, 8]], "qids": ["Q934036"], "gold": [true]}

nq/test_50_bootleg.jsonl

@@ -0,0 +1,50 @@
+{"sentence": "Who did the voice of the magician in Frosty the Snowman", "sent_idx_unq": 0, "aliases": ["voice of", "magician", "frosty the snowman"], "spans": [[3, 5], [6, 7], [8, 11]], "qids": ["Q-1", "Q-1", "Q-1"], "gold": [true, true, true]}
+{"sentence": "What is considered the Outer Banks in North Carolina", "sent_idx_unq": 1, "aliases": ["outer banks", "north carolina"], "spans": [[4, 6], [7, 9]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "The Nashville sound brought a polished and cosmopolitan sound to country music by", "sent_idx_unq": 2, "aliases": ["nashville sound", "music by"], "spans": [[1, 3], [11, 13]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "What channel is the Premier League on in France", "sent_idx_unq": 3, "aliases": ["premier league", "france"], "spans": [[4, 6], [8, 9]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "I Love It ( feat . Charli XCX ) Icona Pop", "sent_idx_unq": 4, "aliases": ["charli xcx", "icona pop"], "spans": [[6, 8], [9, 11]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "The U.S. Supreme Court hears appeals from circuit courts", "sent_idx_unq": 5, "aliases": ["us supreme court", "circuit courts"], "spans": [[1, 4], [7, 9]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Why does the author say that the vampire in Nosferatu is named Count Orlok and not Count Dracula", "sent_idx_unq": 6, "aliases": ["vampire", "nosferatu", "count orlok", "count dracula"], "spans": [[7, 8], [9, 10], [12, 14], [16, 18]], "qids": ["Q-1", "Q-1", "Q-1", "Q-1"], "gold": [true, true, true, true]}
+{"sentence": "Is there an active volcano in New Zealand", "sent_idx_unq": 7, "aliases": ["volcano", "new zealand"], "spans": [[4, 5], [6, 8]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Once Upon a Time Season 6 episode list", "sent_idx_unq": 8, "aliases": ["upon a time", "season 6", "episode list"], "spans": [[1, 4], [4, 6], [6, 8]], "qids": ["Q-1", "Q-1", "Q-1"], "gold": [true, true, true]}
+{"sentence": "Who is the former co-chairman Goldman Sachs who became a U.S. Secretary of the Treasury", "sent_idx_unq": 9, "aliases": ["goldman sachs", "us secretary of the treasury"], "spans": [[5, 7], [10, 15]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Who plays Norman Bates in the TV show", "sent_idx_unq": 10, "aliases": ["norman bates"], "spans": [[2, 4]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Hitchhiker 's Guide to the Galaxy Slartibartfast quotes", "sent_idx_unq": 11, "aliases": ["hitchhiker s guide to the galaxy"], "spans": [[0, 6]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "What was Dennis Hopper 's bike in Easy Rider", "sent_idx_unq": 12, "aliases": ["dennis hopper", "easy rider"], "spans": [[2, 4], [7, 9]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Where was 10 Things I Hate About You filmed school", "sent_idx_unq": 13, "aliases": ["10 things i hate about you"], "spans": [[2, 8]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Where does the last name Aponte come from", "sent_idx_unq": 14, "aliases": ["aponte"], "spans": [[5, 6]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "What teams are in the FA Cup Final", "sent_idx_unq": 15, "aliases": ["fa cup final"], "spans": [[5, 8]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Where did Britain create colonies for its empire", "sent_idx_unq": 16, "aliases": ["britain"], "spans": [[2, 3]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Is it a bank holiday today in Spain", "sent_idx_unq": 17, "aliases": ["bank holiday", "spain"], "spans": [[3, 5], [7, 8]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "1970 World Cup semi final Italy vs Germany", "sent_idx_unq": 18, "aliases": ["1970 world cup", "italy", "germany"], "spans": [[0, 3], [5, 6], [7, 8]], "qids": ["Q-1", "Q-1", "Q-1"], "gold": [true, true, true]}
+{"sentence": "Who played the bank robber in Dirty Harry", "sent_idx_unq": 19, "aliases": ["bank robber", "dirty harry"], "spans": [[3, 5], [6, 8]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "What is the worth of the Catholic Church", "sent_idx_unq": 20, "aliases": ["worth", "catholic church"], "spans": [[3, 4], [6, 8]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "The pair of hand drums used in Indian classical music is called", "sent_idx_unq": 21, "aliases": ["hand drums", "indian classical music"], "spans": [[3, 5], [7, 10]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Game of Thrones Season 1 white hair girl", "sent_idx_unq": 22, "aliases": ["game of thrones season 1", "white hair"], "spans": [[0, 5], [5, 7]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Where is Israel located on the world map", "sent_idx_unq": 23, "aliases": ["israel", "world map"], "spans": [[2, 3], [6, 8]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Who played Smiley in Tinker Tailor Soldier Spy", "sent_idx_unq": 24, "aliases": ["smiley", "tinker tailor soldier spy"], "spans": [[2, 3], [4, 8]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Which of these was not an export of Ancient Greece", "sent_idx_unq": 25, "aliases": ["ancient greece"], "spans": [[8, 10]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Who opened and closed the 1960 Winter Olympics", "sent_idx_unq": 26, "aliases": ["1960 winter olympics"], "spans": [[5, 8]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "I see the river Tiber foaming with much blood", "sent_idx_unq": 27, "aliases": ["river tiber"], "spans": [[3, 5]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "What causes a dead zone in the ocean", "sent_idx_unq": 28, "aliases": ["dead zone"], "spans": [[3, 5]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Who plays Claire Underwood 's mom on House of Cards", "sent_idx_unq": 29, "aliases": ["claire underwood", "mom", "house of cards"], "spans": [[2, 4], [5, 6], [7, 10]], "qids": ["Q-1", "Q-1", "Q-1"], "gold": [true, true, true]}
+{"sentence": "What is the T Rex name in Land Before Time", "sent_idx_unq": 30, "aliases": ["t rex", "time"], "spans": [[3, 5], [9, 10]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Who does Oregon state play in the College World Series", "sent_idx_unq": 31, "aliases": ["oregon state", "college world series"], "spans": [[2, 4], [7, 10]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Who played in the last 3 NBA Finals", "sent_idx_unq": 32, "aliases": ["nba finals"], "spans": [[6, 8]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "UK national debt as percentage of GDP by year", "sent_idx_unq": 33, "aliases": ["uk national debt", "gdp"], "spans": [[0, 3], [6, 7]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Who 's doing the halftime show in 2018", "sent_idx_unq": 34, "aliases": ["halftime show"], "spans": [[4, 6]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Reasons why South Africa should include renewable energy in its energy mix", "sent_idx_unq": 35, "aliases": ["reasons why", "south africa", "renewable energy", "energy mix"], "spans": [[0, 2], [2, 4], [6, 8], [10, 12]], "qids": ["Q-1", "Q-1", "Q-1", "Q-1"], "gold": [true, true, true, true]}
+{"sentence": "When was the first freeway built in Los Angeles", "sent_idx_unq": 36, "aliases": ["freeway", "los angeles"], "spans": [[4, 5], [7, 9]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Landmark Supreme Court cases dealing with the First Amendment", "sent_idx_unq": 37, "aliases": ["supreme court", "first amendment"], "spans": [[1, 3], [7, 9]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "When was the Wizard of Oz made in Technicolor", "sent_idx_unq": 38, "aliases": ["wizard of oz", "technicolor"], "spans": [[3, 6], [8, 9]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Where is the movie Call Me by Your Name filmed", "sent_idx_unq": 39, "aliases": ["call me by your name"], "spans": [[4, 9]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Where does the last name Vigil come from", "sent_idx_unq": 40, "aliases": ["vigil"], "spans": [[5, 6]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Who was President of the United States in 1938", "sent_idx_unq": 41, "aliases": ["president of the united states"], "spans": [[2, 7]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Who proposed the coordinate system to describe the position of a point in a plane accurately", "sent_idx_unq": 42, "aliases": [], "spans": [], "qids": [], "gold": []}
+{"sentence": "When was last time England were in a World Cup semi final", "sent_idx_unq": 43, "aliases": ["time", "england", "cup semi final"], "spans": [[3, 4], [4, 5], [9, 12]], "qids": ["Q-1", "Q-1", "Q-1"], "gold": [true, true, true]}
+{"sentence": "The representative of the British crown in NZ", "sent_idx_unq": 44, "aliases": ["british crown", "nz"], "spans": [[4, 6], [7, 8]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "When did Rangers last win the Scottish Cup", "sent_idx_unq": 45, "aliases": ["rangers", "scottish cup"], "spans": [[2, 3], [6, 8]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Who controls the past controls the future Rage Against the Machine", "sent_idx_unq": 46, "aliases": ["rage against the machine"], "spans": [[7, 11]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "Who is Mariah Carey talking about in We Belong Together", "sent_idx_unq": 47, "aliases": ["mariah carey"], "spans": [[2, 4]], "qids": ["Q-1"], "gold": [true]}
+{"sentence": "What was the Japanese motivation for bombing Pearl Harbor", "sent_idx_unq": 48, "aliases": ["motivation", "pearl harbor"], "spans": [[4, 5], [7, 9]], "qids": ["Q-1", "Q-1"], "gold": [true, true]}
+{"sentence": "Cast of characters in Fiddler on the Roof", "sent_idx_unq": 49, "aliases": ["fiddler on the roof"], "spans": [[4, 8]], "qids": ["Q-1"], "gold": [true]}

nq/test_50_tagme.jsonl

@@ -0,0 +1,50 @@
+{"sentence": "Who did the voice of the magician in Frosty the Snowman", "sent_idx_unq": 0, "aliases": [], "spans": [], "qids": [], "gold": [], "probs": []}
+{"sentence": "What is considered the Outer Banks in North Carolina", "sent_idx_unq": 1, "aliases": ["Outer Banks", "North Carolina"], "spans": [[4, 6], [7, 9]], "qids": ["Q1517373", "Q1454"], "gold": [true, true], "probs": [0.7521328926086426, 0.4903368353843689]}
+{"sentence": "The Nashville sound brought a polished and cosmopolitan sound to country music by", "sent_idx_unq": 2, "aliases": ["Nashville sound", "country music"], "spans": [[1, 3], [10, 12]], "qids": ["Q1751782", "Q83440"], "gold": [true, true], "probs": [0.41436463594436646, 0.448527067899704]}
+{"sentence": "What channel is the Premier League on in France", "sent_idx_unq": 3, "aliases": ["Premier League", "France"], "spans": [[4, 6], [8, 9]], "qids": ["Q9448", "Q47774"], "gold": [true, true], "probs": [0.3656925857067108, 0.3734423518180847]}
+{"sentence": "I Love It ( feat . Charli XCX ) Icona Pop", "sent_idx_unq": 4, "aliases": ["Charli XCX", "Icona Pop"], "spans": [[6, 8], [9, 11]], "qids": ["Q5084390", "Q808703"], "gold": [true, true], "probs": [0.5937708616256714, 0.5]}
+{"sentence": "The U.S. Supreme Court hears appeals from circuit courts", "sent_idx_unq": 5, "aliases": ["U.S. Supreme Court", "Supreme Court", "appeals", "circuit courts"], "spans": [[1, 4], [2, 4], [5, 6], [7, 9]], "qids": ["Q11201", "Q190752", "Q329777", "Q7892368"], "gold": [true, true, true, true], "probs": [0.28778502345085144, 0.4197781980037689, 0.30930158495903015, 0.3313100039958954]}
+{"sentence": "Why does the author say that the vampire in Nosferatu is named Count Orlok and not Count Dracula", "sent_idx_unq": 6, "aliases": ["vampire", "Nosferatu", "Count Orlok", "Count Dracula"], "spans": [[7, 8], [9, 10], [12, 14], [16, 18]], "qids": ["Q46721", "Q151895", "Q1442062", "Q3266236"], "gold": [true, true, true, true], "probs": [0.33512717485427856, 0.8038351535797119, 0.9611549377441406, 0.5]}
+{"sentence": "Is there an active volcano in New Zealand", "sent_idx_unq": 7, "aliases": ["active volcano", "volcano", "New Zealand"], "spans": [[3, 5], [4, 5], [6, 8]], "qids": ["Q8072", "Q8072", "Q664"], "gold": [true, true, true], "probs": [0.3451046347618103, 0.43670186400413513, 0.37270092964172363]}
+{"sentence": "Once Upon a Time Season 6 episode list", "sent_idx_unq": 8, "aliases": ["Once Upon a Time", "episode list"], "spans": [[0, 4], [6, 8]], "qids": ["Q23673", "Q2625600"], "gold": [true, true], "probs": [0.5444839000701904, 0.3404715359210968]}
+{"sentence": "Who is the former co-chairman Goldman Sachs who became a U.S. Secretary of the Treasury", "sent_idx_unq": 9, "aliases": ["Goldman Sachs", "Treasury"], "spans": [[5, 7], [14, 15]], "qids": ["Q193326", "Q648666"], "gold": [true, true], "probs": [0.5680487155914307, 0.25643646717071533]}
+{"sentence": "Who plays Norman Bates in the TV show", "sent_idx_unq": 10, "aliases": ["Norman Bates"], "spans": [[2, 4]], "qids": ["Q2467893"], "gold": [true], "probs": [0.5965998768806458]}
+{"sentence": "Hitchhiker 's Guide to the Galaxy Slartibartfast quotes", "sent_idx_unq": 11, "aliases": ["Hitchhiker 's Guide to the Galaxy", "Slartibartfast"], "spans": [[0, 6], [6, 7]], "qids": ["Q25169", "Q779920"], "gold": [true, true], "probs": [0.5, 0.5]}
+{"sentence": "What was Dennis Hopper 's bike in Easy Rider", "sent_idx_unq": 12, "aliases": ["Dennis Hopper", "bike", "Easy Rider"], "spans": [[2, 4], [5, 6], [7, 9]], "qids": ["Q102711", "Q34493", "Q503638"], "gold": [true, true, true], "probs": [0.5, 0.20313189923763275, 0.7056898474693298]}
+{"sentence": "Where was 10 Things I Hate About You filmed school", "sent_idx_unq": 13, "aliases": ["10 Things I Hate About You"], "spans": [[2, 8]], "qids": ["Q169082"], "gold": [true], "probs": [0.5]}
+{"sentence": "Where does the last name Aponte come from", "sent_idx_unq": 14, "aliases": [], "spans": [], "qids": [], "gold": [], "probs": []}
+{"sentence": "What teams are in the FA Cup Final", "sent_idx_unq": 15, "aliases": ["FA Cup", "FA Cup Final"], "spans": [[5, 7], [5, 8]], "qids": ["Q808791", "Q11151"], "gold": [true, true], "probs": [0.782990574836731, 0.48478567600250244]}
+{"sentence": "Where did Britain create colonies for its empire", "sent_idx_unq": 16, "aliases": ["Britain", "colonies", "its empire"], "spans": [[2, 3], [4, 5], [6, 8]], "qids": ["Q8680", "Q8680", "Q8680"], "gold": [true, true, true], "probs": [0.5378997921943665, 0.5193371176719666, 0.5218750238418579]}
+{"sentence": "Is it a bank holiday today in Spain", "sent_idx_unq": 17, "aliases": ["Spain"], "spans": [[7, 8]], "qids": ["Q29"], "gold": [true], "probs": [0.23816432058811188]}
+{"sentence": "1970 World Cup semi final Italy vs Germany", "sent_idx_unq": 18, "aliases": ["1970 World Cup", "World Cup", "semi final", "Italy", "Germany"], "spans": [[0, 3], [1, 3], [3, 5], [5, 6], [7, 8]], "qids": ["Q132664", "Q19317", "Q864897", "Q676899", "Q183"], "gold": [true, true, true, true, true], "probs": [0.36270493268966675, 0.32496246695518494, 0.2656835913658142, 0.3360447287559509, 0.3292718529701233]}
+{"sentence": "Who played the bank robber in Dirty Harry", "sent_idx_unq": 19, "aliases": ["bank robber", "Dirty Harry"], "spans": [[3, 5], [6, 8]], "qids": ["Q806824", "Q110206"], "gold": [true, true], "probs": [0.3048693835735321, 0.5]}
+{"sentence": "What is the worth of the Catholic Church", "sent_idx_unq": 20, "aliases": ["Catholic Church"], "spans": [[6, 8]], "qids": ["Q9592"], "gold": [true], "probs": [0.2230536937713623]}
+{"sentence": "The pair of hand drums used in Indian classical music is called", "sent_idx_unq": 21, "aliases": ["hand drums", "drums", "Indian classical music"], "spans": [[3, 5], [4, 5], [7, 10]], "qids": ["Q1575231", "Q386854", "Q1323698"], "gold": [true, true, true], "probs": [0.29600754380226135, 0.3520960509777069, 0.25768086314201355]}
+{"sentence": "Game of Thrones Season 1 white hair girl", "sent_idx_unq": 22, "aliases": ["Game of Thrones", "Game of Thrones Season 1"], "spans": [[0, 3], [0, 5]], "qids": ["Q1751870", "Q1658029"], "gold": [true, true], "probs": [0.9667131900787354, 0.8000465631484985]}
+{"sentence": "Where is Israel located on the world map", "sent_idx_unq": 23, "aliases": ["Israel"], "spans": [[2, 3]], "qids": ["Q801"], "gold": [true], "probs": [0.3534795641899109]}
+{"sentence": "Who played Smiley in Tinker Tailor Soldier Spy", "sent_idx_unq": 24, "aliases": ["Smiley", "Tinker Tailor Soldier Spy"], "spans": [[2, 3], [4, 8]], "qids": ["Q2335880", "Q681962"], "gold": [true, true], "probs": [0.2720961570739746, 0.5]}
+{"sentence": "Which of these was not an export of Ancient Greece", "sent_idx_unq": 25, "aliases": ["Ancient Greece", "Greece"], "spans": [[8, 10], [9, 10]], "qids": ["Q11772", "Q11772"], "gold": [true, true], "probs": [0.5667358636856079, 0.5684190988540649]}
+{"sentence": "Who opened and closed the 1960 Winter Olympics", "sent_idx_unq": 26, "aliases": ["1960 Winter Olympics"], "spans": [[5, 8]], "qids": ["Q9634"], "gold": [true], "probs": [0.5]}
+{"sentence": "I see the river Tiber foaming with much blood", "sent_idx_unq": 27, "aliases": ["river Tiber", "Tiber"], "spans": [[3, 5], [4, 5]], "qids": ["Q13712", "Q13712"], "gold": [true, true], "probs": [0.3702380955219269, 0.44560879468917847]}
+{"sentence": "What causes a dead zone in the ocean", "sent_idx_unq": 28, "aliases": ["dead zone", "ocean"], "spans": [[3, 5], [7, 8]], "qids": ["Q1236263", "Q9430"], "gold": [true, true], "probs": [0.646419882774353, 0.23922930657863617]}
+{"sentence": "Who plays Claire Underwood 's mom on House of Cards", "sent_idx_unq": 29, "aliases": ["Claire Underwood", "mom"], "spans": [[2, 4], [5, 6]], "qids": ["Q14915624", "Q13417213"], "gold": [true, true], "probs": [0.8371066451072693, 0.3318559229373932]}
+{"sentence": "What is the T Rex name in Land Before Time", "sent_idx_unq": 30, "aliases": ["T Rex", "Land Before Time"], "spans": [[3, 5], [7, 10]], "qids": ["Q14332", "Q1068384"], "gold": [true, true], "probs": [0.5452110767364502, 0.64427649974823]}
+{"sentence": "Who does Oregon state play in the College World Series", "sent_idx_unq": 31, "aliases": ["Oregon", "Oregon state", "College World Series"], "spans": [[2, 3], [2, 4], [7, 10]], "qids": ["Q7101185", "Q861888", "Q787505"], "gold": [true, true, true], "probs": [0.3431716859340668, 0.36259257793426514, 0.5445585250854492]}
+{"sentence": "Who played in the last 3 NBA Finals", "sent_idx_unq": 32, "aliases": ["NBA Finals"], "spans": [[6, 8]], "qids": ["Q2036161"], "gold": [true], "probs": [0.39057159423828125]}
+{"sentence": "UK national debt as percentage of GDP by year", "sent_idx_unq": 33, "aliases": ["UK national debt", "GDP"], "spans": [[0, 3], [6, 7]], "qids": ["Q611713", "Q12638"], "gold": [true, true], "probs": [0.3333333432674408, 0.3287287950515747]}
+{"sentence": "Who 's doing the halftime show in 2018", "sent_idx_unq": 34, "aliases": ["Who", "Who 's", "halftime show"], "spans": [[0, 1], [0, 2], [4, 6]], "qids": ["Q93346", "Q93346", "Q902899"], "gold": [true, true, true], "probs": [0.3378724753856659, 0.33751174807548523, 0.21875862777233124]}
+{"sentence": "Reasons why South Africa should include renewable energy in its energy mix", "sent_idx_unq": 35, "aliases": ["South Africa"], "spans": [[2, 4]], "qids": ["Q258"], "gold": [true], "probs": [0.40274778008461]}
+{"sentence": "When was the first freeway built in Los Angeles", "sent_idx_unq": 36, "aliases": ["freeway", "Los Angeles"], "spans": [[4, 5], [7, 9]], "qids": ["Q46622", "Q104994"], "gold": [true, true], "probs": [0.31947392225265503, 0.2647697925567627]}
+{"sentence": "Landmark Supreme Court cases dealing with the First Amendment", "sent_idx_unq": 37, "aliases": ["Supreme Court", "Court cases", "First Amendment"], "spans": [[1, 3], [2, 4], [7, 9]], "qids": ["Q11201", "Q2334719", "Q12616"], "gold": [true, true, true], "probs": [0.2926574647426605, 0.22089193761348724, 0.2884783148765564]}
+{"sentence": "When was the Wizard of Oz made in Technicolor", "sent_idx_unq": 38, "aliases": ["Wizard of Oz", "Technicolor"], "spans": [[3, 6], [8, 9]], "qids": ["Q193695", "Q674564"], "gold": [true, true], "probs": [0.7441760301589966, 0.6100959181785583]}
+{"sentence": "Where is the movie Call Me by Your Name filmed", "sent_idx_unq": 39, "aliases": [], "spans": [], "qids": [], "gold": [], "probs": []}
+{"sentence": "Where does the last name Vigil come from", "sent_idx_unq": 40, "aliases": [], "spans": [], "qids": [], "gold": [], "probs": []}
+{"sentence": "Who was President of the United States in 1938", "sent_idx_unq": 41, "aliases": ["President of the United States"], "spans": [[2, 7]], "qids": ["Q11696"], "gold": [true], "probs": [0.3957219421863556]}
+{"sentence": "Who proposed the coordinate system to describe the position of a point in a plane accurately", "sent_idx_unq": 42, "aliases": ["coordinate system", "position", "point", "plane"], "spans": [[3, 5], [8, 9], [11, 12], [14, 15]], "qids": ["Q11210", "Q11210", "Q44946", "Q17285"], "gold": [true, true, true, true], "probs": [0.6101632118225098, 0.4255428910255432, 0.38120409846305847, 0.39132773876190186]}
+{"sentence": "When was last time England were in a World Cup semi final", "sent_idx_unq": 43, "aliases": ["England", "semi final"], "spans": [[4, 5], [10, 12]], "qids": ["Q47762", "Q864897"], "gold": [true, true], "probs": [0.3121379613876343, 0.2656835913658142]}
+{"sentence": "The representative of the British crown in NZ", "sent_idx_unq": 44, "aliases": ["representative", "British crown", "NZ"], "spans": [[1, 2], [4, 6], [7, 8]], "qids": ["Q188759", "Q21941952", "Q1472704"], "gold": [true, true, true], "probs": [0.22304579615592957, 0.3238784670829773, 0.252008855342865]}
+{"sentence": "When did Rangers last win the Scottish Cup", "sent_idx_unq": 45, "aliases": ["Rangers", "Scottish Cup"], "spans": [[2, 3], [6, 8]], "qids": ["Q19597", "Q308822"], "gold": [true, true], "probs": [0.3000677227973938, 0.6878024935722351]}
+{"sentence": "Who controls the past controls the future Rage Against the Machine", "sent_idx_unq": 46, "aliases": ["Rage Against the Machine"], "spans": [[7, 11]], "qids": ["Q72092"], "gold": [true], "probs": [0.5]}
+{"sentence": "Who is Mariah Carey talking about in We Belong Together", "sent_idx_unq": 47, "aliases": ["Who", "Mariah Carey", "We Belong Together"], "spans": [[0, 1], [2, 4], [7, 10]], "qids": ["Q93346", "Q41076", "Q129158"], "gold": [true, true, true], "probs": [0.2421966940164566, 0.5340085029602051, 0.48691099882125854]}
+{"sentence": "What was the Japanese motivation for bombing Pearl Harbor", "sent_idx_unq": 48, "aliases": ["Japanese", "bombing", "Pearl Harbor"], "spans": [[3, 4], [6, 7], [7, 9]], "qids": ["Q188712", "Q52418", "Q52418"], "gold": [true, true, true], "probs": [0.30632221698760986, 0.3024449050426483, 0.6231106519699097]}
+{"sentence": "Cast of characters in Fiddler on the Roof", "sent_idx_unq": 49, "aliases": ["characters", "Fiddler on the Roof"], "spans": [[2, 3], [4, 8]], "qids": ["Q460427", "Q487330"], "gold": [true, true], "probs": [0.28695225715637207, 0.5972776412963867]}