sail_scripts/code-finetuner/data_miner.sh

#!/bin/bash
# Data Miner (Linux - Bash)

echo "=== Data Miner (WSL) ==="
read -p "Enter source (Path or URL): " source
# Remove leading/trailing quotes if user entered them
source="${source%\"}"
source="${source#\"}"
source="${source%\'}"
source="${source#\'}"

# Basic smart detection
if [[ $source == http* ]]; then
    detected_type="url"
elif [[ -d $source ]]; then
    detected_type="dir"
else
    detected_type="jsonl"
fi

echo "Detected type: $detected_type"
read -p "Confirm Type (jsonl, dir, url) [Enter to use $detected_type]: " type
type=${type:-$detected_type}

read -p "Fields (space separated): " fields

echo "Checking dependencies..."
pip install -r requirements.txt || { echo "Pip check failed, attempting with --no-build-isolation"; pip install --no-build-isolation fast_hadamard_transform; }

echo "Starting Data Mining..."
# Handle quotes in path
python3 data_collector.py --source "$source" --type "$type" --fields $fields

echo "Data mining complete. Files organized in ./data/processed"