| { |
| "@context": [ |
| "https://schema.org", |
| { |
| "rai": "http://mlcommons.org/croissant/RAI/", |
| "prov": "http://www.w3.org/ns/prov#", |
| "sc": "https://schema.org/" |
| } |
| ], |
| "@type": "Dataset", |
| "name": "Unified_Farmework", |
| "description": "anonymoususerabcdedg/Unified_Farmework dataset hosted on Hugging Face and contributed by the HF Datasets community", |
| "alternateName": [ |
| "anonymoususerabcdedg/Unified_Farmework" |
| ], |
| "creator": { |
| "@type": "Person", |
| "name": "AnonymousUser", |
| "url": "https://huggingface.co/anonymoususerabcdedg" |
| }, |
| "keywords": [ |
| "🇺🇸 Region: US" |
| ], |
| "url": "https://huggingface.co/datasets/anonymoususerabcdedg/Unified_Farmework", |
| "rai:dataLimitations": "This dataset is reconstructed from existing open-source agent benchmarks into a unified triplet format defined in our paper: instruction set, toolkit set, and environment set. Known limitations: (1) Coverage is bounded by the original source benchmarks, which are predominantly English and focused on academic/research scenarios rather than authentic user interactions. (2) Some tasks depend on external tools or live environments that may change over time, affecting reproducibility. (3) The reformatting into the unified triplet schema may introduce minor information loss compared to the original benchmark formats. Not recommended for: training agent models (risk of evaluation contamination), or as the sole basis for production deployment decisions.", |
| "rai:dataBiases": "Biases inherent to the source benchmarks are inherited by this dataset: (1) Language and geographic skew: source benchmarks are predominantly English and authored by research groups in North America, Europe, and East Asia, leading to overrepresentation of Western digital workflows (e.g., e-commerce, English-language web navigation, US-style customer service in TauBench) and underrepresentation of other languages, regions, and cultural contexts. (2) Selection bias in tasks: instructions, tools, and environments were designed by AI researchers rather than sampled from real user behavior, so the distribution reflects what researchers believe agents should be tested on (tool calling, web browsing, code, multi-agent coordination) rather than authentic end-user demand. (3) Synthetic data bias: portions of the source benchmarks (e.g., TauBench, parts of AgentBench) are generated by frontier LLMs, and therefore inherit the stylistic, cultural, and reasoning biases of those generator models. (4) Domain skew toward digital/tech scenarios; physical-world, low-resource professional, and non-English-speaking-user scenarios are largely absent. Models evaluated or trained on this dataset may therefore perform better on Western, English, digitally-mediated agent tasks and worse on underrepresented settings.", |
| "rai:personalSensitiveInformation": "The dataset includes webpage snapshots crawled from public websites as environment data, which may incidentally contain the following categories of information: (1) Geography — webpages reflect the geographic origin of their hosts and content (predominantly US/English-language sites); (2) Language — primarily English, with some other languages present in source benchmarks; (3) Culture — webpage content reflects the cultural context of the source sites; (4) Incidental personal information — public webpages may include names, usernames, public profile photos, or other publicly posted information of identifiable individuals (e.g., authors of articles, public figures, forum users). No private, non-public, or deliberately collected personal data (such as health records, political/religious beliefs, socio-economic status, age, or gender labels) is included. All webpage snapshots are sourced from publicly accessible URLs, and the dataset does not augment or annotate any personal attributes beyond what was already public on the source pages. Users of this dataset should be aware that webpage snapshots are static copies and may preserve information that has since been removed from the live web.", |
| "rai:dataUseCases": "Construct: This dataset is intended to measure the agentic capability of LLM. Validity has been established for: (1) evaluating and comparing LLM agent systems on tool use, planning, web browsing, multi-agent collaboration, and safety; (2) cross-benchmark analysis enabled by the unified schema; (3) ablation studies on agent components (instruction following vs. tool selection vs. environment interaction). Validity has NOT been established for: (1) training agent models on this data — doing so would contaminate the benchmarks for future evaluation;", |
| "rai:dataSocialImpact": "Positive impact: by unifying fragmented agent benchmarks into a common (instruction, toolkit, environment) format, this dataset lowers the barrier to reproducible agent evaluation, enables fairer cross-system comparison, and supports the development of safer and more capable LLM agents. Negative impact and risks: (1) Misuse for training — if used as training data, it may contaminate future evaluations and inflate reported agent capabilities. (2) Overgeneralization — strong performance on this dataset does not guarantee safe or reliable behavior in real-world deployment, and stakeholders may overestimate agent readiness. (3) Fairness implications — because the data is predominantly English and reflects Western digital workflows, agents tuned to perform well here may underserve users in other languages, regions, and cultural contexts. Mitigations: we explicitly document these limitations and biases (see corresponding RAI fields), recommend against using the data for training, honor takedown requests for any incidentally included webpage content, and release the dataset under a license that restricts use to research and evaluation purposes.", |
| "rai:hasSyntheticData": true, |
| "prov:wasDerivedFrom": [ |
| { |
| "@id": "https://github.com/THUDM/AgentBench", |
| "prov:label": "AgentBench", |
| "sc:license": "Apache License 2.0" |
| }, |
| { |
| "@id": "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard", |
| "prov:label": "BFCL", |
| "sc:license": "Apache License 2.0" |
| }, |
| { |
| "@id": "https://github.com/sierra-research/tau-bench", |
| "prov:label": "tau-bench", |
| "sc:license": "MIT License" |
| }, |
| { |
| "@id": "https://github.com/sierra-research/tau2-bench", |
| "prov:label": "tau2-bench", |
| "sc:license": "MIT License" |
| }, |
| { |
| "@id": "https://github.com/openai/simple-evals", |
| "prov:label": "BrowseComp", |
| "sc:license": "MIT License" |
| } |
| ], |
| "prov:wasGeneratedBy": [ |
| { |
| "@type": "prov:Activity", |
| "prov:type": { |
| "@id": "https://www.wikidata.org/wiki/Q4929239" |
| }, |
| "prov:label": "Data Collection", |
| "sc:description": "Source agent benchmarkswere downloaded from their official public GitHub and Hugging Face repositories. No additional human-collected data was introduced." |
| }, |
| { |
| "@type": "prov:Activity", |
| "prov:type": { |
| "@id": "https://www.wikidata.org/wiki/Q109719325" |
| }, |
| "prov:label": "No new label annotations added", |
| "sc:description": "No new human or LLM annotations were added; all task labels and ground-truth answers are inherited from the source benchmarks." |
| }, |
| { |
| "@type": "prov:Activity", |
| "prov:type": { |
| "@id": "https://www.wikidata.org/wiki/Q3306762" |
| }, |
| "prov:label": "Data verification", |
| "sc:description": "All processing was performed by the authors; no crowdsourcing platforms or external annotators were used." |
| }, |
| { |
| "@type": "prov:Activity", |
| "prov:type": { |
| "@id": "https://www.wikidata.org/wiki/Q5227332" |
| }, |
| "prov:label": "Data Formatting", |
| "sc:description": "Each source benchmark was parsed and reformatted into the unified (instruction, toolkit, environment) triplet schema defined in our paper." |
| } |
| ], |
| "license": "CC BY 4.0" |
| } |
