File size: 2,479 Bytes
9064fbe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | AttributeError Traceback (most recent call last)
<ipython-input-15-cf6a974cddb7> in <cell line: 124>()
330 print(f'Running evaluation on {eval_tasks} with {num_fewshot}-shot examples')
331
--> 332 results = adapter.run_eval(
333 eval_tasks=eval_tasks,
334 num_fewshot=num_fewshot,
/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py in decorate_context(*args, **kwargs)
114 def decorate_context(*args, **kwargs):
115 with ctx_factory():
--> 116 return func(*args, **kwargs)
117
118 return decorate_context
<ipython-input-15-cf6a974cddb7> in run_eval(self, eval_tasks, num_fewshot, limit, bootstrap_iters, fewshot_random_seed)
312 task_obj.set_config(key="num_fewshot", value=num_fewshot)
313
--> 314 results = evaluator.evaluate(
315 lm=self,
316 task_dict=task_dict,
/usr/local/lib/python3.10/dist-packages/lm_eval/utils.py in _wrapper(*args, **kwargs)
393 "lm-evaluation-harness!"
394 )
--> 395 return fn(*args, **kwargs)
396
397 return _wrapper
/usr/local/lib/python3.10/dist-packages/lm_eval/evaluator.py in evaluate(lm, task_dict, limit, cache_requests, rewrite_requests_cache, bootstrap_iters, write_out, log_samples, system_instruction, apply_chat_template, fewshot_as_multiturn, verbosity)
447
448 # run requests through model
--> 449 resps = getattr(lm, reqtype)(cloned_reqs)
450
451 # put responses from model into a list of length K for each request.
/usr/local/lib/python3.10/dist-packages/lm_eval/api/model.py in loglikelihood(self, requests, disable_tqdm)
361 # BOS or EOS as context
362 context_enc, continuation_enc = (
--> 363 [self.prefix_token_id],
364 self.tok_encode(continuation),
365 )
/usr/local/lib/python3.10/dist-packages/lm_eval/models/huggingface.py in prefix_token_id(self)
360 def prefix_token_id(self):
361 # it is used as prefix for loglikelihood
--> 362 if self.custom_prefix_token_id is not None:
363 return self.custom_prefix_token_id
364 if self.tokenizer.bos_token_id is not None:
AttributeError: 'EvalHarnessAdapter' object has no attribute 'custom_prefix_token_id' |