| import abc | |
| from abc import abstractmethod | |
| class SequenceParallel(abc.ABC): | |
| def init_sequence_parallel(self, size): | |
| pass | |
| def prepare_model(self, model, tokenizer, split_in_forward): | |
| pass | |
| def pad_and_split_inputs(self, | |
| tokenizer, | |
| input_ids, | |
| input_embeds, | |
| labels, | |
| position_ids, | |
| attention_mask, | |
| loss_scale, | |
| embed_tokens=None): | |
| pass | |
| def reduce_outputs(self, loss, labels): | |
| pass | |
| def sp_group(self): | |
| return None | |
| def world_size(self): | |
| pass | |
| def prepare_trainer(self, trainer): | |
| pass | |
| def get_dataloader(self, trainer, dataset, batch_size): | |
| pass | |