{ "meta": { "name": "Pay Equity RAG Benchmark v1", "created": "2026-06-14", "description": "15 questions testing both EU Directive 2023/970 retrieval and IDA salary statistics retrieval. Each question has ground-truth chunk IDs that contain the relevant answer information, plus a list of answer facts the generation should contain.", "num_questions": 15, "directive_questions": 9, "salary_questions": 6 }, "questions": [ { "id": "Q01", "type": "directive", "question": "Can a Danish employer ask me about my previous salary during a job interview?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-5"], "answer_facts": [ "An employer shall not ask applicants about their pay history", "This is from Article 5 of the EU Pay Transparency Directive", "Job vacancy notices and job titles must be gender-neutral" ], "notes": "Tests retrieval of Art. 5 (pay transparency prior to employment) — specifically paragraph 2 which prohibits asking about pay history." }, { "id": "Q02", "type": "directive", "question": "What are my rights as a worker to see what my colleagues earn?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-7"], "answer_facts": [ "Workers have the right to request and receive in writing information on their individual pay level", "Average pay levels broken down by sex for categories of workers performing the same work", "Must be provided within two months from the date of the request", "Employers shall inform all workers on an annual basis of their right to receive this information", "Workers shall not be prevented from disclosing their pay" ], "notes": "Tests retrieval of Art. 7 (right to information). Key facts: 2-month deadline, annual notification obligation, can go through workers' representatives." }, { "id": "Q03", "type": "directive", "question": "At what company size does mandatory pay gap reporting kick in under the new EU directive?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-9"], "answer_facts": [ "Employers with at least 100 workers should regularly report on pay", "Reports must include the gender pay gap", "Reports must include the gender pay gap in complementary or variable components", "Reports must include the median gender pay gap", "Reports must include the proportion of female and male workers receiving complementary or variable components", "Reports must include the proportion of female and male workers in each quartile pay band" ], "notes": "Tests retrieval of Art. 9 (reporting on pay gap). The 100-worker threshold is the key fact." }, { "id": "Q04", "type": "directive", "question": "What happens if a company's pay report shows women earn less than men — does the directive require specific action?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-10"], "answer_facts": [ "Joint pay assessment must be carried out when the difference in average pay level is at least 5%", "Triggered in any given category of workers", "If the employer has not remedied the difference within six months of the pay report submission", "Carried out by employers in cooperation with workers' representatives", "Joint pay assessments should lead to elimination of gender-based pay discrimination through remedial measures" ], "notes": "Tests retrieval of Art. 10 (joint pay assessment). The 5% gap trigger and 6-month deadline are key." }, { "id": "Q05", "type": "directive", "question": "If I sue my employer for pay discrimination and they didn't follow the transparency rules, who has to prove what?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-18"], "answer_facts": [ "The burden of proof shifts to the employer (respondent)", "This happens when the employer does not comply with pay transparency obligations", "Based on the Danfoss case (C-109/88)", "Employer must prove the infringement was manifestly unintentional and of a minor character to avoid the shift" ], "notes": "Tests retrieval of Art. 18 (shift of burden of proof). Key concept: burden shifts to employer on non-compliance." }, { "id": "Q06", "type": "directive", "question": "How long do I have to bring an equal pay claim under the EU Pay Transparency Directive?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-21"], "answer_facts": [ "Limitation period for bringing claims is at least three years", "Period does not begin to run before the claimant is aware of the infringement", "Member States can decide the period does not begin while the infringement is ongoing", "Member States can decide it does not begin before the end of the employment contract" ], "notes": "Tests retrieval of Art. 21 (limitation periods). 3 years minimum, awareness-based trigger." }, { "id": "Q07", "type": "directive", "question": "What counts as 'pay' under the directive — just salary or also bonuses and other benefits?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-3"], "answer_facts": [ "'Pay' means the ordinary basic or minimum wage or salary and any other consideration", "Includes complementary or variable components", "Can be in cash or in kind", "Includes bonuses, overtime compensation, travel facilities, housing and food allowances", "Includes compensation for attending training, payments in case of dismissal, statutory sick pay", "Includes occupational pensions", "Includes all elements of remuneration due under law, collective agreements and practice" ], "notes": "Tests retrieval of Art. 3 definitions. Pay is defined broadly. This is a definition-lookup question." }, { "id": "Q08", "type": "directive", "question": "What is the deadline for EU member states to implement the Pay Transparency Directive into national law?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-34"], "answer_facts": [ "Deadline is 7 June 2026", "Member States shall bring into force the laws, regulations and administrative provisions", "They shall immediately inform the Commission" ], "notes": "Tests retrieval of Art. 34 (transposition). Specific date." }, { "id": "Q09", "type": "directive", "question": "What penalties can be imposed on employers who violate the equal pay principle under the directive?", "lang": "en", "ground_truth_chunk_ids": ["dir-art-23"], "answer_facts": [ "Penalties must be effective, proportionate and dissuasive", "Fines should be possible", "Fines could be based on the employer's gross annual turnover or total payroll", "Repeated infringements should face specific penalties", "Financial disincentives such as revocation of public benefits or exclusion from public tender procedures" ], "notes": "Tests retrieval of Art. 23 (penalties)." }, { "id": "Q10", "type": "salary", "question": "Hvad er medianlønnen for en civilingeniør i det private med 10 års erfaring ifølge IDAs lønstatistik for 2025?", "lang": "da", "ground_truth_chunk_ids": ["ida-p011-0035"], "answer_facts": [ "Median salary is 67,870 DKK/month", "Average salary is 68,387 DKK/month", "25th percentile is 61,222 DKK/month", "75th percentile is 74,086 DKK/month", "90th percentile is 79,669 DKK/month", "Source: IDA Lønstatistik 2025, p.11", "For 2015 graduation year (~10 years experience)" ], "notes": "Tests retrieval of civilingeniør salary data for 10yr experience (2015 graduation). Key: median=67,870. Danish-language question." }, { "id": "Q11", "type": "salary", "question": "What is the median salary for a civilingeniør (MSc engineer) with 5 years of experience in the Danish private sector? How many respondents and what percentage receive a bonus?", "lang": "en", "ground_truth_chunk_ids": ["ida-p011-0040"], "answer_facts": [ "Median salary is 57,256 DKK/month", "Average salary is 58,581 DKK/month", "25th percentile is 52,936 DKK/month", "75th percentile is 63,283 DKK/month", "90th percentile is 69,472 DKK/month", "40.2% receive bonus", "Average bonus is 61,482 DKK/year", "Based on 420 respondents", "Source: IDA Lønstatistik 2025, p.11" ], "notes": "Tests retrieval of civilingeniør salary for 5yr experience (2020 graduation). Large sample size (420 respondents) should be cited." }, { "id": "Q12", "type": "salary", "question": "What is the average starting salary for a newly graduated civilingeniør (2025 graduate with 0 years experience) in the Danish private sector?", "lang": "en", "ground_truth_chunk_ids": ["ida-p011-0045"], "answer_facts": [ "Average salary is 47,775 DKK/month", "Median salary is 47,118 DKK/month", "25th percentile is 44,396 DKK/month", "75th percentile is 50,586 DKK/month", "90th percentile is 53,038 DKK/month", "Source: IDA Lønstatistik 2025, p.11" ], "notes": "Tests retrieval of civilingeniør starting salary (2025 graduation)." }, { "id": "Q13", "type": "salary", "question": "What is the average salary for a topchef/funktionsdirektør (C-level executive) with 25 years of experience according to IDA 2025 salary statistics?", "lang": "en", "ground_truth_chunk_ids": ["ida-p021-1371"], "answer_facts": [ "Average salary is 115,018 DKK/month", "For Topchef/Funktionsdirektør with ~25 years experience (2000 graduation)", "Source: IDA Lønstatistik 2025, p.21", "This is in the private sector" ], "notes": "Tests retrieval from page 21 (Stillingstype/position type). Tests cross-referencing position type with experience. 2000 graduation = ~25 yrs." }, { "id": "Q14", "type": "salary", "question": "What is the average salary in the pharmaceutical industry (medicinalindustri) for IDA engineers with 5 years of experience?", "lang": "en", "ground_truth_chunk_ids": ["ida-p017-1176"], "answer_facts": [ "Average salary is 62,654 DKK/month", "For Medicinalindustri (pharmaceutical industry)", "For 2020 graduation year (~5 years experience)", "Source: IDA Lønstatistik 2025, p.17" ], "notes": "Tests retrieval from page 17 (Branche/industry segmentation). Cross-reference: industry + experience. 2020 = ~5yrs." }, { "id": "Q15", "type": "salary", "question": "What is the average salary for an IDA engineer working in the state sector (staten) with around 20 years of experience?", "lang": "en", "ground_truth_chunk_ids": ["ida-p030-1698"], "answer_facts": [ "Average salary is 65,167 DKK/month", "For 2005 graduation year (~20 years experience)", "Based on 163 respondents", "Source: IDA Lønstatistik 2025, p.30", "State sector (Staten: Stillingsniveau)" ], "notes": "Tests retrieval from page 30 (state sector by stillingsniveau/position level). 2005 graduation = ~20yrs experience. Average 65,167 vs private sector civilingeniør 82,569 shows public-private gap." } ] }