jinaai
/

jina-bert-implementation

Model card Files Files and versions

Fix sorting heuristic

#3

by Markus28 - opened Oct 17, 2023

base: refs/heads/main

←

from: refs/pr/3

Discussion Files changed

Files changed (1) hide show

modeling_bert.py +3 -2

modeling_bert.py CHANGED Viewed

@@ -1211,8 +1211,9 @@ class JinaBertModel(JinaBertPreTrainedModel):
             self.to(device)
         # TODO: Maybe use better length heuristic?
-        inverse_permutation = np.argsort(np.argsort([-len(i) for i in sentences]))
-        sentences = sorted(sentences, key=len, reverse=True)
         padding = tokenizer_kwargs.pop('padding', True)

             self.to(device)
         # TODO: Maybe use better length heuristic?
+        permutation = np.argsort([-len(i) for i in sentences])
+        inverse_permutation = np.argsort(permutation)
+        sentences = [sentences[idx] for idx in permutation]
         padding = tokenizer_kwargs.pop('padding', True)