#!/bin/bash set -euo pipefail CORPUS_NAME="${1:?usage: $0 }" REPO_URL="${REPO_URL:-https://github.com/vedatonuryilmaz/dev_genopix.git}" REPO_BRANCH="${REPO_BRANCH:-ml-intern}" DATA_DATASET="${DATA_DATASET:-vedatonuryilmaz/te-variant-simulator-v1}" REFERENCE_DATASET="${REFERENCE_DATASET:-vedatonuryilmaz/te-seqdata-v1}" REFERENCE_REVISION="${REFERENCE_REVISION:-f107d46c5a61087eb3fb19b4e3e75fdef0b74fdd}" REFERENCE_FILE="${REFERENCE_FILE:-train/te_seqdata.parquet}" REFERENCE_PARQUET="${REFERENCE_PARQUET:-data/raw/reference/${REFERENCE_FILE}}" CORPUS_KIND="${CORPUS_KIND:-controlled}" N_LOCI="${N_LOCI:-64}" COVERAGE="${COVERAGE:-50}" READ_LENGTH="${READ_LENGTH:-150}" ERROR_RATE="${ERROR_RATE:-0.001}" BP_PER_TOKEN="${BP_PER_TOKEN:-24}" VARIANT_TYPES="${VARIANT_TYPES:-snp}" if [ "${DEEPGENOPIX_ALREADY_CLONED:-0}" = "1" ]; then echo "using existing DeepGenopix checkout at $(pwd)" else git clone --branch "$REPO_BRANCH" --depth 1 "$REPO_URL" deepgenopix cd deepgenopix fi uv sync --extra hf --extra quant if [ ! -f "$REFERENCE_PARQUET" ]; then mkdir -p "$(dirname "$REFERENCE_PARQUET")" uv run python -c " from huggingface_hub import hf_hub_download hf_hub_download( repo_id='$REFERENCE_DATASET', repo_type='dataset', revision='$REFERENCE_REVISION', filename='$REFERENCE_FILE', local_dir='data/raw/reference', ) " fi OUTPUT_DIR="data/output/simulator/$CORPUS_KIND/$CORPUS_NAME" mkdir -p "$OUTPUT_DIR" uv run python -c " from pathlib import Path from deepgenopix.variant.simulate import SimulatorConfig, materialize_corpus variant_types = tuple(item.strip() for item in '$VARIANT_TYPES'.split(',') if item.strip()) cfg = SimulatorConfig( coverage=int('$COVERAGE'), read_length=int('$READ_LENGTH'), error_rate=float('$ERROR_RATE'), bp_per_token=int('$BP_PER_TOKEN'), variant_types=variant_types, ) paths = materialize_corpus( Path('$REFERENCE_PARQUET'), cfg, Path('$OUTPUT_DIR'), n_loci=int('$N_LOCI'), ) print(paths) " uv run python scripts/hf_artifacts.py push-dataset-folder \ --repo "$DATA_DATASET" \ --source "$OUTPUT_DIR" \ --path-in-repo "$CORPUS_KIND/$CORPUS_NAME" \ --message "variant simulator corpus $CORPUS_KIND/$CORPUS_NAME"