sort?

app.py

@@ -5,7 +5,7 @@ import glob
 import tqdm
 import torch
 import torch.nn.functional as F
-from transformers import AutoTokenizer, AutoModel
+from transformers import AutoTokenizer, AutoModel, set_seed
 from peft import PeftModel
 from tevatron.retriever.searcher import FaissFlatSearcher
 import logging
@@ -20,6 +20,8 @@ import peft
 import faiss
 import sys
 
+set_seed(42)
+
 # Set up logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -176,12 +178,15 @@ def load_corpus_lookups(dataset_name):
     global corpus_lookups
     corpus_path = f"{dataset_name}/corpus_emb.*.pkl"
     index_files = glob.glob(corpus_path)
+    # sort them
+    index_files.sort(key=lambda x: int(x.split('.')[-2]))
     
     corpus_lookups[dataset_name] = []
     for file in index_files:
         with open(file, 'rb') as f:
             _, p_lookup = pickle.load(f)
         corpus_lookups[dataset_name] += p_lookup
+
     logger.info(f"Loaded corpus lookups for {dataset_name}. Total entries: {len(corpus_lookups[dataset_name])}")
     logger.info(f"Sample corpus lookup entry: {corpus_lookups[dataset_name][0]}")