experiments/bioagent_bench/results/by_task/transcript-quant.json

{
  "evaluated_at_unix": 1778738056.609411,
  "task_id": "transcript-quant",
  "runs_root": "/225040511/project/Hypo_Bio_OS/bioagent-bench-runs",
  "dataset_root": "/225040511/project/bioagent-bench/dataset",
  "manifest": {
    "mcp_root": "/225040511/project/Biomni/mcp_generated",
    "execution_python": "/225040511/miniconda3/envs/biomni_e1/bin/python",
    "available_server_count": 793,
    "available_tool_count": 3189,
    "targets": {
      "100": {
        "config": "/225040511/project/Biomni/experiments/mcp_tool_scaling/configs/mcp_scale_100.yaml",
        "server_count": 10,
        "tool_count": 107,
        "servers": [
          "mcp_kallisto",
          "mcp_kraken2",
          "mcp_csvtk",
          "mcp_megahit",
          "mcp_kaiju",
          "mcp_fastp",
          "mcp_spades",
          "mcp_scanpy",
          "mcp_scanpy-cli",
          "mcp_scanpy-scripts"
        ]
      },
      "500": {
        "config": "/225040511/project/Biomni/experiments/mcp_tool_scaling/configs/mcp_scale_500.yaml",
        "server_count": 96,
        "tool_count": 504,
        "servers": [
          "mcp_kallisto",
          "mcp_kraken2",
          "mcp_csvtk",
          "mcp_megahit",
          "mcp_kaiju",
          "mcp_fastp",
          "mcp_spades",
          "mcp_scanpy",
          "mcp_scanpy-cli",
          "mcp_scanpy-scripts",
          "mcp_seurat-scripts",
          "mcp_bioconductor-deseq2",
          "mcp_bioconductor-edger",
          "mcp_bioconductor-limma",
          "mcp_bioconductor-clusterprofiler",
          "mcp_star",
          "mcp_hisat2",
          "mcp_subread",
          "mcp_bcftools",
          "mcp_bwa",
          "mcp_gatk4",
          "mcp_gatk",
          "mcp_snpeff",
          "mcp_freebayes",
          "mcp_orthofinder",
          "mcp_blast",
          "mcp_blast-legacy",
          "mcp_diamond",
          "mcp_gffread",
          "mcp_seqkit",
          "mcp_multiqc",
          "mcp_fastqc",
          "mcp_trimmomatic",
          "mcp_abnumber",
          "mcp_abricate",
          "mcp_abundancebin",
          "mcp_abyss",
          "mcp_adapterremoval",
          "mcp_alfred",
          "mcp_anansescanpy",
          "mcp_anarci",
          "mcp_aragorn",
          "mcp_aria2",
          "mcp_ariba",
          "mcp_arriba",
          "mcp_art",
          "mcp_arvados-cwl-runner",
          "mcp_arvados-python-client",
          "mcp_atropos",
          "mcp_augustus",
          "mcp_auspice",
          "mcp_bactopia",
          "mcp_bamtools",
          "mcp_barrnap",
          "mcp_bbmap",
          "mcp_bcbio-gff",
          "mcp_bcbio-nextgen",
          "mcp_bedops",
          "mcp_bedtools",
          "mcp_bin2cell",
          "mcp_bioawk",
          "mcp_biobambam",
          "mcp_bioconductor-affy",
          "mcp_bioconductor-affyio",
          "mcp_bioconductor-alabaster",
          "mcp_bioconductor-alabaster.sfe",
          "mcp_bioconductor-alabaster.spatial",
          "mcp_bioconductor-annotate",
          "mcp_bioconductor-annotationdbi",
          "mcp_bioconductor-annotationfilter",
          "mcp_bioconductor-apeglm",
          "mcp_bioconductor-apl",
          "mcp_bioconductor-awaggregator",
          "mcp_bioconductor-banksy",
          "mcp_bioconductor-beachmat",
          "mcp_bioconductor-benchdamic",
          "mcp_bioconductor-biobase",
          "mcp_bioconductor-biocbaseutils",
          "mcp_bioconductor-biocfilecache",
          "mcp_bioconductor-biocgenerics",
          "mcp_bioconductor-biocio",
          "mcp_bioconductor-biocneighbors",
          "mcp_bioconductor-biocparallel",
          "mcp_bioconductor-biocsingular",
          "mcp_bioconductor-biomart",
          "mcp_bioconductor-biomformat",
          "mcp_bioconductor-biostrings",
          "mcp_bioconductor-biovizbase",
          "mcp_bioconductor-blase",
          "mcp_bioconductor-bluster",
          "mcp_bioconductor-bulksignalr",
          "mcp_bioconductor-cardspa",
          "mcp_bioconductor-catscradle",
          "mcp_bioconductor-cdi",
          "mcp_bioconductor-cellhashr",
          "mcp_bioconductor-cellid"
        ]
      },
      "1000": {
        "config": "/225040511/project/Biomni/experiments/mcp_tool_scaling/configs/mcp_scale_1000.yaml",
        "server_count": 275,
        "tool_count": 1002,
        "servers": [
          "mcp_kallisto",
          "mcp_kraken2",
          "mcp_csvtk",
          "mcp_megahit",
          "mcp_kaiju",
          "mcp_fastp",
          "mcp_spades",
          "mcp_scanpy",
          "mcp_scanpy-cli",
          "mcp_scanpy-scripts",
          "mcp_seurat-scripts",
          "mcp_bioconductor-deseq2",
          "mcp_bioconductor-edger",
          "mcp_bioconductor-limma",
          "mcp_bioconductor-clusterprofiler",
          "mcp_star",
          "mcp_hisat2",
          "mcp_subread",
          "mcp_bcftools",
          "mcp_bwa",
          "mcp_gatk4",
          "mcp_gatk",
          "mcp_snpeff",
          "mcp_freebayes",
          "mcp_orthofinder",
          "mcp_blast",
          "mcp_blast-legacy",
          "mcp_diamond",
          "mcp_gffread",
          "mcp_seqkit",
          "mcp_multiqc",
          "mcp_fastqc",
          "mcp_trimmomatic",
          "mcp_abnumber",
          "mcp_abricate",
          "mcp_abundancebin",
          "mcp_abyss",
          "mcp_adapterremoval",
          "mcp_alfred",
          "mcp_anansescanpy",
          "mcp_anarci",
          "mcp_aragorn",
          "mcp_aria2",
          "mcp_ariba",
          "mcp_arriba",
          "mcp_art",
          "mcp_arvados-cwl-runner",
          "mcp_arvados-python-client",
          "mcp_atropos",
          "mcp_augustus",
          "mcp_auspice",
          "mcp_bactopia",
          "mcp_bamtools",
          "mcp_barrnap",
          "mcp_bbmap",
          "mcp_bcbio-gff",
          "mcp_bcbio-nextgen",
          "mcp_bedops",
          "mcp_bedtools",
          "mcp_bin2cell",
          "mcp_bioawk",
          "mcp_biobambam",
          "mcp_bioconductor-affy",
          "mcp_bioconductor-affyio",
          "mcp_bioconductor-alabaster",
          "mcp_bioconductor-alabaster.sfe",
          "mcp_bioconductor-alabaster.spatial",
          "mcp_bioconductor-annotate",
          "mcp_bioconductor-annotationdbi",
          "mcp_bioconductor-annotationfilter",
          "mcp_bioconductor-apeglm",
          "mcp_bioconductor-apl",
          "mcp_bioconductor-awaggregator",
          "mcp_bioconductor-banksy",
          "mcp_bioconductor-beachmat",
          "mcp_bioconductor-benchdamic",
          "mcp_bioconductor-biobase",
          "mcp_bioconductor-biocbaseutils",
          "mcp_bioconductor-biocfilecache",
          "mcp_bioconductor-biocgenerics",
          "mcp_bioconductor-biocio",
          "mcp_bioconductor-biocneighbors",
          "mcp_bioconductor-biocparallel",
          "mcp_bioconductor-biocsingular",
          "mcp_bioconductor-biomart",
          "mcp_bioconductor-biomformat",
          "mcp_bioconductor-biostrings",
          "mcp_bioconductor-biovizbase",
          "mcp_bioconductor-blase",
          "mcp_bioconductor-bluster",
          "mcp_bioconductor-bulksignalr",
          "mcp_bioconductor-cardspa",
          "mcp_bioconductor-catscradle",
          "mcp_bioconductor-cdi",
          "mcp_bioconductor-cellhashr",
          "mcp_bioconductor-cellid",
          "mcp_bioconductor-cellmigration",
          "mcp_bioconductor-clusterfoldsimilarity",
          "mcp_bioconductor-clustifyr",
          "mcp_bioconductor-clustsignal",
          "mcp_bioconductor-complexheatmap",
          "mcp_bioconductor-concordexr",
          "mcp_bioconductor-cotan",
          "mcp_bioconductor-csoa",
          "mcp_bioconductor-ctsv",
          "mcp_bioconductor-curatedatlasqueryr",
          "mcp_bioconductor-cytomapper",
          "mcp_bioconductor-dada2",
          "mcp_bioconductor-data-packages",
          "mcp_bioconductor-decipher",
          "mcp_bioconductor-decontam",
          "mcp_bioconductor-decontx",
          "mcp_bioconductor-deconvobuddies",
          "mcp_bioconductor-delayedmatrixstats",
          "mcp_bioconductor-despace",
          "mcp_bioconductor-diffbind",
          "mcp_bioconductor-dino",
          "mcp_bioconductor-dirichletmultinomial",
          "mcp_bioconductor-dnacopy",
          "mcp_bioconductor-dose",
          "mcp_bioconductor-ensembldb",
          "mcp_bioconductor-erma",
          "mcp_bioconductor-escher",
          "mcp_bioconductor-experimentsubset",
          "mcp_bioconductor-fgsea",
          "mcp_bioconductor-genefilter",
          "mcp_bioconductor-geneplotter",
          "mcp_bioconductor-genomeinfodb",
          "mcp_bioconductor-genomeinfodbdata",
          "mcp_bioconductor-genomicalignments",
          "mcp_bioconductor-genomicfeatures",
          "mcp_bioconductor-genomicranges",
          "mcp_bioconductor-geomxtools",
          "mcp_bioconductor-ggsc",
          "mcp_bioconductor-ggspavis",
          "mcp_bioconductor-ggtree",
          "mcp_bioconductor-glmgampoi",
          "mcp_bioconductor-go.db",
          "mcp_bioconductor-gosemsim",
          "mcp_bioconductor-graph",
          "mcp_bioconductor-gsva",
          "mcp_bioconductor-hcatonsildata",
          "mcp_bioconductor-hdf5array",
          "mcp_bioconductor-hoodscanr",
          "mcp_bioconductor-humanhippocampus2024",
          "mcp_bioconductor-imcdatasets",
          "mcp_bioconductor-imcrtools",
          "mcp_bioconductor-impute",
          "mcp_bioconductor-infercnv",
          "mcp_bioconductor-interactivedisplaybase",
          "mcp_bioconductor-iranges",
          "mcp_bioconductor-irisfgm",
          "mcp_bioconductor-jazzpanda",
          "mcp_bioconductor-keggrest",
          "mcp_bioconductor-lisaclust",
          "mcp_bioconductor-mastr",
          "mcp_bioconductor-matrixgenerics",
          "mcp_bioconductor-merfishdata",
          "mcp_bioconductor-metabolomicsworkbenchr",
          "mcp_bioconductor-metapod",
          "mcp_bioconductor-mia",
          "mcp_bioconductor-moleculeexperiment",
          "mcp_bioconductor-mosim",
          "mcp_bioconductor-mousegastrulationdata",
          "mcp_bioconductor-mspurity",
          "mcp_bioconductor-multiassayexperiment",
          "mcp_bioconductor-multtest",
          "mcp_bioconductor-nebulosa",
          "mcp_bioconductor-nnsvg",
          "mcp_bioconductor-noiseq",
          "mcp_bioconductor-orfhunter",
          "mcp_bioconductor-org.ce.eg.db",
          "mcp_bioconductor-org.hs.eg.db",
          "mcp_bioconductor-org.mm.eg.db",
          "mcp_bioconductor-partcnv",
          "mcp_bioconductor-phemd",
          "mcp_bioconductor-phyloseq",
          "mcp_bioconductor-pipecomp",
          "mcp_bioconductor-poem",
          "mcp_bioconductor-preprocesscore",
          "mcp_bioconductor-proteomicsannotationhubdata",
          "mcp_bioconductor-protgenerics",
          "mcp_bioconductor-qvalue",
          "mcp_bioconductor-rbgl",
          "mcp_bioconductor-reactomegsa.data",
          "mcp_bioconductor-regionalst",
          "mcp_bioconductor-rforproteomics",
          "mcp_bioconductor-rgraphviz",
          "mcp_bioconductor-rhdf5",
          "mcp_bioconductor-rhdf5filters",
          "mcp_bioconductor-rhdf5lib",
          "mcp_bioconductor-rhtslib",
          "mcp_bioconductor-rsamtools",
          "mcp_bioconductor-rsubread",
          "mcp_bioconductor-scaledmatrix",
          "mcp_bioconductor-scalign",
          "mcp_bioconductor-scarray.sat",
          "mcp_bioconductor-scbfa",
          "mcp_bioconductor-scbubbletree",
          "mcp_bioconductor-sccb2",
          "mcp_bioconductor-scclassifr",
          "mcp_bioconductor-scdataviz",
          "mcp_bioconductor-scdotplot",
          "mcp_bioconductor-scfeatures",
          "mcp_bioconductor-scider",
          "mcp_bioconductor-scmageck",
          "mcp_bioconductor-scqtltools",
          "mcp_bioconductor-scran",
          "mcp_bioconductor-screpertoire",
          "mcp_bioconductor-scrnaseq",
          "mcp_bioconductor-sctreeviz",
          "mcp_bioconductor-scuttle",
          "mcp_bioconductor-scvir",
          "mcp_bioconductor-seraster",
          "mcp_bioconductor-shortread",
          "mcp_bioconductor-signifinder",
          "mcp_bioconductor-singlecellmultimodal",
          "mcp_bioconductor-smoothclust",
          "mcp_bioconductor-smoppix",
          "mcp_bioconductor-sosta",
          "mcp_bioconductor-spacetrooper",
          "mcp_bioconductor-spaniel",
          "mcp_bioconductor-spari",
          "mcp_bioconductor-sparsematrixstats",
          "mcp_bioconductor-spatialcpie",
          "mcp_bioconductor-spatialdatasets",
          "mcp_bioconductor-spatialdecon",
          "mcp_bioconductor-spatialdmelxsim",
          "mcp_bioconductor-spatialexperiment",
          "mcp_bioconductor-spatialexperimentio",
          "mcp_bioconductor-spatialfda",
          "mcp_bioconductor-spatialfeatureexperiment",
          "mcp_bioconductor-spatialheatmap",
          "mcp_bioconductor-speckle",
          "mcp_bioconductor-spoon",
          "mcp_bioconductor-spotclean",
          "mcp_bioconductor-spotsweeper",
          "mcp_bioconductor-standr",
          "mcp_bioconductor-statial",
          "mcp_bioconductor-stjoincount",
          "mcp_bioconductor-summarizedexperiment",
          "mcp_bioconductor-svp",
          "mcp_bioconductor-tenxxeniumdata",
          "mcp_bioconductor-tidyomics",
          "mcp_bioconductor-tidyspatialexperiment",
          "mcp_bioconductor-treesummarizedexperiment",
          "mcp_bioconductor-tximport",
          "mcp_bioconductor-variantannotation",
          "mcp_bioconductor-vectrapolarisdata",
          "mcp_bioconductor-visiumio",
          "mcp_bioconductor-weberdivechalcdata",
          "mcp_bioconductor-xcms",
          "mcp_bioconductor-xenlite",
          "mcp_bioconductor-xvector",
          "mcp_bioconductor-zlibbioc",
          "mcp_biopython",
          "mcp_bowtie",
          "mcp_bowtie2",
          "mcp_bpipe",
          "mcp_brooklyn_plot",
          "mcp_busco",
          "mcp_bx-python",
          "mcp_c-ares",
          "mcp_cascade-reg",
          "mcp_cd-hit",
          "mcp_cdbtools",
          "mcp_cell2cell",
          "mcp_cellitac",
          "mcp_cellqc",
          "mcp_cellrank",
          "mcp_cellsnake",
          "mcp_celltypist",
          "mcp_celltypist-so",
          "mcp_checkatlas",
          "mcp_checkm-genome"
        ]
      }
    }
  },
  "aggregate": {
    "run_count": 2,
    "mean_retrieval_recall@1": 0.25,
    "mean_retrieval_recall@3": 0.5,
    "mean_retrieval_recall@5": 1.0,
    "mean_retrieval_recall@10": 1.0,
    "mean_retrieval_recall@20": 1.0,
    "mean_retrieval_recall@50": 1.0,
    "tool_selection_accuracy": 1.0,
    "workflow_validity": 1.0,
    "execution_success_rate": 1.0,
    "mean_context_tokens_rough": 900.0,
    "mean_planning_latency_seconds": null
  },
  "results": [
    {
      "task_id": "transcript-quant",
      "run_dir": "/225040511/project/Hypo_Bio_OS/bioagent-bench-runs/transcript-quant_20260513_065059",
      "retrieval_recall_at_k": {
        "1": 0.0,
        "3": 0.5,
        "5": 1.0,
        "10": 1.0,
        "20": 1.0,
        "50": 1.0
      },
      "tool_selection_accuracy": true,
      "workflow_validity": {
        "valid": true,
        "steps_matched": 4,
        "steps_total": 4,
        "matched_step_names": [
          "build_transcriptome_index",
          "quantify_paired_reads",
          "extract_counts",
          "write_tsv"
        ]
      },
      "execution_success": true,
      "context_tokens_rough": 886,
      "planning_latency_seconds": null,
      "retrieved_candidate_count": 612,
      "execution_evidence_count": 2510
    },
    {
      "task_id": "transcript-quant",
      "run_dir": "/225040511/project/Hypo_Bio_OS/bioagent-bench-runs/transcript-quant_20260513_105632",
      "retrieval_recall_at_k": {
        "1": 0.5,
        "3": 0.5,
        "5": 1.0,
        "10": 1.0,
        "20": 1.0,
        "50": 1.0
      },
      "tool_selection_accuracy": true,
      "workflow_validity": {
        "valid": true,
        "steps_matched": 4,
        "steps_total": 4,
        "matched_step_names": [
          "build_transcriptome_index",
          "quantify_paired_reads",
          "extract_counts",
          "write_tsv"
        ]
      },
      "execution_success": true,
      "context_tokens_rough": 914,
      "planning_latency_seconds": null,
      "retrieved_candidate_count": 619,
      "execution_evidence_count": 2801
    }
  ]
}