| AttributeError Traceback (most recent call last) | |
| <ipython-input-15-cf6a974cddb7> in <cell line: 124>() | |
| 330 print(f'Running evaluation on {eval_tasks} with {num_fewshot}-shot examples') | |
| 331 | |
| --> 332 results = adapter.run_eval( | |
| 333 eval_tasks=eval_tasks, | |
| 334 num_fewshot=num_fewshot, | |
| /usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py in decorate_context(*args, **kwargs) | |
| 114 def decorate_context(*args, **kwargs): | |
| 115 with ctx_factory(): | |
| --> 116 return func(*args, **kwargs) | |
| 117 | |
| 118 return decorate_context | |
| <ipython-input-15-cf6a974cddb7> in run_eval(self, eval_tasks, num_fewshot, limit, bootstrap_iters, fewshot_random_seed) | |
| 312 task_obj.set_config(key="num_fewshot", value=num_fewshot) | |
| 313 | |
| --> 314 results = evaluator.evaluate( | |
| 315 lm=self, | |
| 316 task_dict=task_dict, | |
| /usr/local/lib/python3.10/dist-packages/lm_eval/utils.py in _wrapper(*args, **kwargs) | |
| 393 "lm-evaluation-harness!" | |
| 394 ) | |
| --> 395 return fn(*args, **kwargs) | |
| 396 | |
| 397 return _wrapper | |
| /usr/local/lib/python3.10/dist-packages/lm_eval/evaluator.py in evaluate(lm, task_dict, limit, cache_requests, rewrite_requests_cache, bootstrap_iters, write_out, log_samples, system_instruction, apply_chat_template, fewshot_as_multiturn, verbosity) | |
| 447 | |
| 448 # run requests through model | |
| --> 449 resps = getattr(lm, reqtype)(cloned_reqs) | |
| 450 | |
| 451 # put responses from model into a list of length K for each request. | |
| /usr/local/lib/python3.10/dist-packages/lm_eval/api/model.py in loglikelihood(self, requests, disable_tqdm) | |
| 361 # BOS or EOS as context | |
| 362 context_enc, continuation_enc = ( | |
| --> 363 [self.prefix_token_id], | |
| 364 self.tok_encode(continuation), | |
| 365 ) | |
| /usr/local/lib/python3.10/dist-packages/lm_eval/models/huggingface.py in prefix_token_id(self) | |
| 360 def prefix_token_id(self): | |
| 361 # it is used as prefix for loglikelihood | |
| --> 362 if self.custom_prefix_token_id is not None: | |
| 363 return self.custom_prefix_token_id | |
| 364 if self.tokenizer.bos_token_id is not None: | |
| AttributeError: 'EvalHarnessAdapter' object has no attribute 'custom_prefix_token_id' |