File size: 1,148 Bytes
4ca4e4c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | from models import (
HybridNoPEForCausalLM,
HybridForCausalLM,
HybridNoPEConfig,
HybridConfig
)
def get_model(args, tokenizer):
if not args.nope:
config = HybridConfig(
layers=args.layers,
bos_token_id=0,
eos_token_id=0,
hidden_size=args.hidden_size,
intermediate_size=args.hidden_size*4,
num_attention_heads=args.heads,
d_model=args.hidden_size,
ssm_cfg={"d_state": args.state_dim},
vocab_size=len(tokenizer),
)
if args.nope:
config = HybridNoPEConfig(
layers=args.layers,
bos_token_id=0,
eos_token_id=0,
hidden_size=args.hidden_size,
intermediate_size=args.hidden_size*4,
num_attention_heads=args.heads,
d_model=args.hidden_size,
ssm_cfg={"d_state": args.state_dim},
vocab_size=len(tokenizer),
)
if not args.nope:
model = HybridForCausalLM(config)
if args.nope:
model = HybridNoPEForCausalLM(config)
return model
|