import os import subprocess import json import logging from huggingface_hub import HfApi, create_repo, upload_folder logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s' ) logger = logging.getLogger("TrainingWrapper") def main(): logger.info("Starting training script wrapper in Hugging Face Space.") hf_token_for_push = os.environ.get('HF_TOKEN_FOR_PUSH') if not hf_token_for_push: logger.error("HF_TOKEN_FOR_PUSH secret not found. Cannot upload results.") exit(1) target_model_repo_id = os.environ.get('TARGET_MODEL_REPO_ID') if not target_model_repo_id: logger.error("TARGET_MODEL_REPO_ID environment variable not found.") exit(1) data_path = os.environ.get('DATA_PATH_FOR_SCRIPT') base_model_id = os.environ.get('BASE_MODEL_ID_FOR_SCRIPT') if not data_path or not base_model_id: logger.error("DATA_PATH_FOR_SCRIPT or BASE_MODEL_ID_FOR_SCRIPT env vars missing.") exit(1) hyperparameters_json_str = os.getenv('HYPERPARAMETERS_JSON_FOR_SCRIPT', '{}') training_script_config_json_str = os.getenv('TRAINING_SCRIPT_CONFIG_JSON_FOR_SCRIPT', '{}') model_output_dir = "/app/outputs" os.makedirs(model_output_dir, exist_ok=True) cmd = [ "python", "train_text_lora.py", "--data_path", data_path, "--model_output_dir", model_output_dir, "--base_model_id", base_model_id, "--hyperparameters_json", hyperparameters_json_str, "--training_script_config_json", training_script_config_json_str, # Add --runner_environment huggingface if train_text_lora.py uses it "--runner_environment", "huggingface" ] logger.info(f"Constructed training command: {' '.join(cmd)}") logger.info(f"Hyperparameters JSON for script: {hyperparameters_json_str}") logger.info(f"Training Script Config JSON for script: {training_script_config_json_str}") logger.info("Executing train_text_lora.py...") # Stream stdout/stderr directly for Space logs process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) if process.stdout: for line in iter(process.stdout.readline, ''): logger.info(line.strip()) # Log each line as it comes process.stdout.close() return_code = process.wait() if return_code == 0: logger.info("Training script completed successfully.") logger.info(f"Uploading model outputs from {model_output_dir} to HF Hub repository: {target_model_repo_id}") try: api = HfApi(token=hf_token_for_push) # Create target repo if it doesn't exist. Privacy should be handled by the runner ideally. create_repo(target_model_repo_id, token=hf_token_for_push, repo_type="model", exist_ok=True) upload_folder( folder_path=model_output_dir, repo_id=target_model_repo_id, repo_type="model", commit_message=f"Job completed: Upload fine-tuned LoRA adapter and artifacts from Space.", token=hf_token_for_push ) logger.info(f"Successfully uploaded artifacts to Hugging Face Hub model repo: {target_model_repo_id}") except Exception as e: logger.error(f"Failed to upload results to {target_model_repo_id}: {e}", exc_info=True) exit(1) # Consider upload failure as a job failure else: logger.error(f"Training script failed with return code {return_code}.") exit(return_code) if __name__ == "__main__": main()