File size: 2,313 Bytes
18a82fb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | #!/usr/bin/env python
"""Setup pipeline for product design year classification"""
import argparse
from pathlib import Path
import subprocess
import sys
import os
def run_step(script_name, args=[]):
"""Run a script step"""
# Get the directory where this script is located
script_dir = Path(__file__).parent
script_path = script_dir / script_name
cmd = [sys.executable, str(script_path)] + args
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print(f"Error in {script_name}:")
print(result.stderr)
print(result.stdout) # Also print stdout for more info
return False
print(result.stdout) # Show the output from the script
return True
def main():
parser = argparse.ArgumentParser(description='Setup data pipeline for product year classification')
parser.add_argument('--skip-validation', action='store_true', help='Skip URL validation')
parser.add_argument('--xlsx-path', default='../data/metadata/fetch_ALL.xlsx', help='Path to XLSX file')
parser.add_argument('--output-dir', default='../data', help='Output directory')
args = parser.parse_args()
print("=== Product Design Year Classification Pipeline ===\n")
# Step 1: Process XLSX
print("Step 1: Processing product data...")
if not run_step('process_xlsx.py'):
return
# Step 2: Validate URLs (optional)
if not args.skip_validation:
print("\nStep 2: Validating image URLs...")
if not run_step('validate_urls.py'):
print("URL validation failed, continuing anyway...")
# Step 3: Create splits
print("\nStep 3: Creating train/val/test splits...")
if not run_step('create_splits.py'):
return
# Step 4: Analyze dataset
print("\nStep 4: Analyzing dataset...")
if not run_step('analyze_dataset.py'):
print("Analysis failed, but pipeline complete")
print("\n=== Pipeline Complete! ===")
print(f"Your data is ready in {args.output_dir}/")
print("\nNext steps:")
print("1. Review the analysis in data/analysis/")
print("2. Check data/metadata/processing_issues.txt for any problems")
print("3. Start training with: python scripts/train.py --model_name efficientnet-b2")
if __name__ == "__main__":
main() |