sail_scripts/code-finetuner/convert_grammar.py

import json
import os

def convert_gec_to_sail(input_path, output_path):
    print(f"Converting {input_path} to {output_path}...")
    sail_data = []
    
    with open(input_path, 'r', encoding='utf-8') as f:
        for line in f:
            try:
                data = json.loads(line)
                # Map GEC fields to SAIL instruction format
                # source is tokens, target is tokens
                instruction = "Correct the grammar in the following text:"
                source_text = " ".join(data['source']) if isinstance(data['source'], list) else data['source']
                target_text = " ".join(data['target']) if isinstance(data['target'], list) else data['target']
                
                # Combine into the SAIL instruction format
                sail_data.append({
                    "instruction": f"{instruction}\n\n{source_text}",
                    "response": target_text,
                    "category": "english_grammar"
                })
            except Exception as e:
                print(f"Error processing line: {e}")
                
    with open(output_path, 'w', encoding='utf-8') as f:
        json.dump(sail_data, f, indent=2)
    
    print(f"Successfully converted {len(sail_data)} samples.")

if __name__ == "__main__":
    input_file = "/mnt/e/agent/agent_ai/code-finetuner/data/processed/grammar/source/collected_data.jsonl"
    output_file = "/mnt/e/agent/agent_ai/datasets/finetune/grammar_instructions.json"
    
    # Ensure output directory exists
    os.makedirs(os.path.dirname(output_file), exist_ok=True)
    
    convert_gec_to_sail(input_file, output_file)