| AttributeError Traceback (most recent call last) |
| <ipython-input-15-cf6a974cddb7> in <cell line: 124>() |
| 330 print(f'Running evaluation on {eval_tasks} with {num_fewshot}-shot examples') |
| 331 |
| --> 332 results = adapter.run_eval( |
| 333 eval_tasks=eval_tasks, |
| 334 num_fewshot=num_fewshot, |
|
|
| /usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py in decorate_context(*args, **kwargs) |
| 114 def decorate_context(*args, **kwargs): |
| 115 with ctx_factory(): |
| --> 116 return func(*args, **kwargs) |
| 117 |
| 118 return decorate_context |
|
|
| <ipython-input-15-cf6a974cddb7> in run_eval(self, eval_tasks, num_fewshot, limit, bootstrap_iters, fewshot_random_seed) |
| 312 task_obj.set_config(key="num_fewshot", value=num_fewshot) |
| 313 |
| --> 314 results = evaluator.evaluate( |
| 315 lm=self, |
| 316 task_dict=task_dict, |
|
|
| /usr/local/lib/python3.10/dist-packages/lm_eval/utils.py in _wrapper(*args, **kwargs) |
| 393 "lm-evaluation-harness!" |
| 394 ) |
| --> 395 return fn(*args, **kwargs) |
| 396 |
| 397 return _wrapper |
|
|
| /usr/local/lib/python3.10/dist-packages/lm_eval/evaluator.py in evaluate(lm, task_dict, limit, cache_requests, rewrite_requests_cache, bootstrap_iters, write_out, log_samples, system_instruction, apply_chat_template, fewshot_as_multiturn, verbosity) |
| 447 |
| 448 # run requests through model |
| --> 449 resps = getattr(lm, reqtype)(cloned_reqs) |
| 450 |
| 451 # put responses from model into a list of length K for each request. |
|
|
| /usr/local/lib/python3.10/dist-packages/lm_eval/api/model.py in loglikelihood(self, requests, disable_tqdm) |
| 361 # BOS or EOS as context |
| 362 context_enc, continuation_enc = ( |
| --> 363 [self.prefix_token_id], |
| 364 self.tok_encode(continuation), |
| 365 ) |
|
|
| /usr/local/lib/python3.10/dist-packages/lm_eval/models/huggingface.py in prefix_token_id(self) |
| 360 def prefix_token_id(self): |
| 361 # it is used as prefix for loglikelihood |
| --> 362 if self.custom_prefix_token_id is not None: |
| 363 return self.custom_prefix_token_id |
| 364 if self.tokenizer.bos_token_id is not None: |
|
|
| AttributeError: 'EvalHarnessAdapter' object has no attribute 'custom_prefix_token_id' |
