| { |
| "meta": { |
| "name": "Pay Equity RAG Benchmark v1", |
| "created": "2026-06-14", |
| "description": "15 questions testing both EU Directive 2023/970 retrieval and IDA salary statistics retrieval. Each question has ground-truth chunk IDs that contain the relevant answer information, plus a list of answer facts the generation should contain.", |
| "num_questions": 15, |
| "directive_questions": 9, |
| "salary_questions": 6 |
| }, |
| "questions": [ |
| { |
| "id": "Q01", |
| "type": "directive", |
| "question": "Can a Danish employer ask me about my previous salary during a job interview?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-5"], |
| "answer_facts": [ |
| "An employer shall not ask applicants about their pay history", |
| "This is from Article 5 of the EU Pay Transparency Directive", |
| "Job vacancy notices and job titles must be gender-neutral" |
| ], |
| "notes": "Tests retrieval of Art. 5 (pay transparency prior to employment) — specifically paragraph 2 which prohibits asking about pay history." |
| }, |
| { |
| "id": "Q02", |
| "type": "directive", |
| "question": "What are my rights as a worker to see what my colleagues earn?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-7"], |
| "answer_facts": [ |
| "Workers have the right to request and receive in writing information on their individual pay level", |
| "Average pay levels broken down by sex for categories of workers performing the same work", |
| "Must be provided within two months from the date of the request", |
| "Employers shall inform all workers on an annual basis of their right to receive this information", |
| "Workers shall not be prevented from disclosing their pay" |
| ], |
| "notes": "Tests retrieval of Art. 7 (right to information). Key facts: 2-month deadline, annual notification obligation, can go through workers' representatives." |
| }, |
| { |
| "id": "Q03", |
| "type": "directive", |
| "question": "At what company size does mandatory pay gap reporting kick in under the new EU directive?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-9"], |
| "answer_facts": [ |
| "Employers with at least 100 workers should regularly report on pay", |
| "Reports must include the gender pay gap", |
| "Reports must include the gender pay gap in complementary or variable components", |
| "Reports must include the median gender pay gap", |
| "Reports must include the proportion of female and male workers receiving complementary or variable components", |
| "Reports must include the proportion of female and male workers in each quartile pay band" |
| ], |
| "notes": "Tests retrieval of Art. 9 (reporting on pay gap). The 100-worker threshold is the key fact." |
| }, |
| { |
| "id": "Q04", |
| "type": "directive", |
| "question": "What happens if a company's pay report shows women earn less than men — does the directive require specific action?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-10"], |
| "answer_facts": [ |
| "Joint pay assessment must be carried out when the difference in average pay level is at least 5%", |
| "Triggered in any given category of workers", |
| "If the employer has not remedied the difference within six months of the pay report submission", |
| "Carried out by employers in cooperation with workers' representatives", |
| "Joint pay assessments should lead to elimination of gender-based pay discrimination through remedial measures" |
| ], |
| "notes": "Tests retrieval of Art. 10 (joint pay assessment). The 5% gap trigger and 6-month deadline are key." |
| }, |
| { |
| "id": "Q05", |
| "type": "directive", |
| "question": "If I sue my employer for pay discrimination and they didn't follow the transparency rules, who has to prove what?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-18"], |
| "answer_facts": [ |
| "The burden of proof shifts to the employer (respondent)", |
| "This happens when the employer does not comply with pay transparency obligations", |
| "Based on the Danfoss case (C-109/88)", |
| "Employer must prove the infringement was manifestly unintentional and of a minor character to avoid the shift" |
| ], |
| "notes": "Tests retrieval of Art. 18 (shift of burden of proof). Key concept: burden shifts to employer on non-compliance." |
| }, |
| { |
| "id": "Q06", |
| "type": "directive", |
| "question": "How long do I have to bring an equal pay claim under the EU Pay Transparency Directive?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-21"], |
| "answer_facts": [ |
| "Limitation period for bringing claims is at least three years", |
| "Period does not begin to run before the claimant is aware of the infringement", |
| "Member States can decide the period does not begin while the infringement is ongoing", |
| "Member States can decide it does not begin before the end of the employment contract" |
| ], |
| "notes": "Tests retrieval of Art. 21 (limitation periods). 3 years minimum, awareness-based trigger." |
| }, |
| { |
| "id": "Q07", |
| "type": "directive", |
| "question": "What counts as 'pay' under the directive — just salary or also bonuses and other benefits?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-3"], |
| "answer_facts": [ |
| "'Pay' means the ordinary basic or minimum wage or salary and any other consideration", |
| "Includes complementary or variable components", |
| "Can be in cash or in kind", |
| "Includes bonuses, overtime compensation, travel facilities, housing and food allowances", |
| "Includes compensation for attending training, payments in case of dismissal, statutory sick pay", |
| "Includes occupational pensions", |
| "Includes all elements of remuneration due under law, collective agreements and practice" |
| ], |
| "notes": "Tests retrieval of Art. 3 definitions. Pay is defined broadly. This is a definition-lookup question." |
| }, |
| { |
| "id": "Q08", |
| "type": "directive", |
| "question": "What is the deadline for EU member states to implement the Pay Transparency Directive into national law?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-34"], |
| "answer_facts": [ |
| "Deadline is 7 June 2026", |
| "Member States shall bring into force the laws, regulations and administrative provisions", |
| "They shall immediately inform the Commission" |
| ], |
| "notes": "Tests retrieval of Art. 34 (transposition). Specific date." |
| }, |
| { |
| "id": "Q09", |
| "type": "directive", |
| "question": "What penalties can be imposed on employers who violate the equal pay principle under the directive?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["dir-art-23"], |
| "answer_facts": [ |
| "Penalties must be effective, proportionate and dissuasive", |
| "Fines should be possible", |
| "Fines could be based on the employer's gross annual turnover or total payroll", |
| "Repeated infringements should face specific penalties", |
| "Financial disincentives such as revocation of public benefits or exclusion from public tender procedures" |
| ], |
| "notes": "Tests retrieval of Art. 23 (penalties)." |
| }, |
| { |
| "id": "Q10", |
| "type": "salary", |
| "question": "Hvad er medianlønnen for en civilingeniør i det private med 10 års erfaring ifølge IDAs lønstatistik for 2025?", |
| "lang": "da", |
| "ground_truth_chunk_ids": ["ida-p011-0035"], |
| "answer_facts": [ |
| "Median salary is 67,870 DKK/month", |
| "Average salary is 68,387 DKK/month", |
| "25th percentile is 61,222 DKK/month", |
| "75th percentile is 74,086 DKK/month", |
| "90th percentile is 79,669 DKK/month", |
| "Source: IDA Lønstatistik 2025, p.11", |
| "For 2015 graduation year (~10 years experience)" |
| ], |
| "notes": "Tests retrieval of civilingeniør salary data for 10yr experience (2015 graduation). Key: median=67,870. Danish-language question." |
| }, |
| { |
| "id": "Q11", |
| "type": "salary", |
| "question": "What is the median salary for a civilingeniør (MSc engineer) with 5 years of experience in the Danish private sector? How many respondents and what percentage receive a bonus?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["ida-p011-0040"], |
| "answer_facts": [ |
| "Median salary is 57,256 DKK/month", |
| "Average salary is 58,581 DKK/month", |
| "25th percentile is 52,936 DKK/month", |
| "75th percentile is 63,283 DKK/month", |
| "90th percentile is 69,472 DKK/month", |
| "40.2% receive bonus", |
| "Average bonus is 61,482 DKK/year", |
| "Based on 420 respondents", |
| "Source: IDA Lønstatistik 2025, p.11" |
| ], |
| "notes": "Tests retrieval of civilingeniør salary for 5yr experience (2020 graduation). Large sample size (420 respondents) should be cited." |
| }, |
| { |
| "id": "Q12", |
| "type": "salary", |
| "question": "What is the average starting salary for a newly graduated civilingeniør (2025 graduate with 0 years experience) in the Danish private sector?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["ida-p011-0045"], |
| "answer_facts": [ |
| "Average salary is 47,775 DKK/month", |
| "Median salary is 47,118 DKK/month", |
| "25th percentile is 44,396 DKK/month", |
| "75th percentile is 50,586 DKK/month", |
| "90th percentile is 53,038 DKK/month", |
| "Source: IDA Lønstatistik 2025, p.11" |
| ], |
| "notes": "Tests retrieval of civilingeniør starting salary (2025 graduation)." |
| }, |
| { |
| "id": "Q13", |
| "type": "salary", |
| "question": "What is the average salary for a topchef/funktionsdirektør (C-level executive) with 25 years of experience according to IDA 2025 salary statistics?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["ida-p021-1371"], |
| "answer_facts": [ |
| "Average salary is 115,018 DKK/month", |
| "For Topchef/Funktionsdirektør with ~25 years experience (2000 graduation)", |
| "Source: IDA Lønstatistik 2025, p.21", |
| "This is in the private sector" |
| ], |
| "notes": "Tests retrieval from page 21 (Stillingstype/position type). Tests cross-referencing position type with experience. 2000 graduation = ~25 yrs." |
| }, |
| { |
| "id": "Q14", |
| "type": "salary", |
| "question": "What is the average salary in the pharmaceutical industry (medicinalindustri) for IDA engineers with 5 years of experience?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["ida-p017-1176"], |
| "answer_facts": [ |
| "Average salary is 62,654 DKK/month", |
| "For Medicinalindustri (pharmaceutical industry)", |
| "For 2020 graduation year (~5 years experience)", |
| "Source: IDA Lønstatistik 2025, p.17" |
| ], |
| "notes": "Tests retrieval from page 17 (Branche/industry segmentation). Cross-reference: industry + experience. 2020 = ~5yrs." |
| }, |
| { |
| "id": "Q15", |
| "type": "salary", |
| "question": "What is the average salary for an IDA engineer working in the state sector (staten) with around 20 years of experience?", |
| "lang": "en", |
| "ground_truth_chunk_ids": ["ida-p030-1698"], |
| "answer_facts": [ |
| "Average salary is 65,167 DKK/month", |
| "For 2005 graduation year (~20 years experience)", |
| "Based on 163 respondents", |
| "Source: IDA Lønstatistik 2025, p.30", |
| "State sector (Staten: Stillingsniveau)" |
| ], |
| "notes": "Tests retrieval from page 30 (state sector by stillingsniveau/position level). 2005 graduation = ~20yrs experience. Average 65,167 vs private sector civilingeniør 82,569 shows public-private gap." |
| } |
| ] |
| } |
|
|
