import os
from pprint import pprint

import torch

os.environ['CUDA_VISIBLE_DEVICES'] = '1'
kwargs = {
    'per_device_train_batch_size': 4,
    'per_device_eval_batch_size': 4,
    'gradient_accumulation_steps': 4,
    'num_train_epochs': 1,
    'save_steps': 100,
    'max_length': 8192,
}


def calc_acc(infer_result):
    n_correct = 0
    for res in infer_result:
        if res['response'] == res['labels']:
            n_correct += 1
    return f'acc: {n_correct/len(infer_result)}, n_correct: {n_correct}, len(res): {len(infer_result)}'


def calc_diff(infer_result, infer_result2):
    n_correct = 0
    for x1, x2 in zip(infer_result, infer_result2):
        if x1['response'] == x2['response']:
            n_correct += 1
    return f'acc: {n_correct/len(infer_result)}, n_correct: {n_correct}, len(res): {len(infer_result)}'


def test_llm():
    from swift.llm import sft_main, TrainArguments, infer_main, InferArguments, Template
    res = []
    for padding_side in ['left', 'right']:
        model = 'Qwen/Qwen2.5-0.5B-Instruct'
        dataset = ['damo/zh_cls_fudan-news#2000']
        result = sft_main(
            TrainArguments(model=model, dataset=dataset, split_dataset_ratio=0.1, padding_side=padding_side, **kwargs))
        last_model_checkpoint = result['last_model_checkpoint']
        infer_result = infer_main(InferArguments(ckpt_dir=last_model_checkpoint, load_data_args=True))
        res.append(calc_acc(infer_result))
        infer_result2 = infer_main(
            InferArguments(ckpt_dir=last_model_checkpoint, load_data_args=True, max_batch_size=16))
        res.append(calc_acc(infer_result2))
    pprint(res)


def test_mllm():
    from swift.llm import sft_main, TrainArguments, infer_main, InferArguments, Template
    res = []
    for padding_side in ['left', 'right']:
        model = 'Qwen/Qwen2-VL-2B-Instruct'
        dataset = ['AI-ModelScope/LaTeX_OCR#2000']
        result = sft_main(TrainArguments(model=model, dataset=dataset, padding_side=padding_side, **kwargs))
        last_model_checkpoint = result['last_model_checkpoint']
        infer_result = infer_main(InferArguments(ckpt_dir=last_model_checkpoint, load_data_args=True))
        res.append(infer_result)
        infer_result2 = infer_main(
            InferArguments(ckpt_dir=last_model_checkpoint, load_data_args=True, max_batch_size=16))
        res.append(infer_result2)
    print(calc_diff(res[0], res[1]))
    print(calc_diff(res[2], res[3]))
    print(calc_diff(res[0], res[2]))
    print(calc_diff(res[0], res[3]))
    print(calc_diff(res[2], res[1]))


if __name__ == '__main__':
    test_llm()
    test_mllm()