Upload folder using huggingface_hub

README.md

@@ -1,10 +1,18 @@
 ---
-title: LudoBench Test
-emoji: 🚀
-colorFrom: red
-colorTo: gray
+title: "LudoBench: Board Game Reasoning Benchmark"
+emoji: "\U0001F3B2"
+colorFrom: blue
+colorTo: purple
 sdk: static
 pinned: false
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# LudoBench
+
+A multimodal board-game reasoning benchmark evaluating LLM/VLM reasoning across 5 strategy games and 3 difficulty tiers.
+
+- 638 annotated QA pairs
+- 5 games: Kingdomino, Res Arcana, Pax Renaissance, Carcassonne, Catan
+- 3 tiers: Environment Perception, Rules Integration, Short-Horizon Optimization
+- 9 models benchmarked across 3 modalities (None, Text, Image)

annotation_data/carcassonne_tier1/test_2001.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many tiles are present in the given landscape?",
+  "Answer": "9",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2001
+}

annotation_data/carcassonne_tier1/test_2002.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "What is the size of the minimum rectangle that can enclose all the tiles in the given landscape? Provide the answer in the format 'width x height'.",
+  "Answer": "4 x 4",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2002
+}

annotation_data/carcassonne_tier1/test_2003.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the current landscape, how many tiles have roads shown on them?",
+  "Answer": "6",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2003
+}

annotation_data/carcassonne_tier1/test_2004.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the current landscape, how many tiles have grass present on them?",
+  "Answer": "8",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2004
+}

annotation_data/carcassonne_tier1/test_2005.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the current landscape, how many tiles contain only grass on them?",
+  "Answer": "0",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2005
+}

annotation_data/carcassonne_tier1/test_2006.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the landscape, how many tiles contain a shield (or coat of arms) symbol?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2006
+}

annotation_data/carcassonne_tier1/test_2007.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many tiles are present in the given landscape?",
+  "Answer": "6",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2007
+}

annotation_data/carcassonne_tier1/test_2008.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the current landscape, how many tiles do not contain any roads?",
+  "Answer": "0",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2008
+}

annotation_data/carcassonne_tier1/test_2009.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the current landscape, how many tiles contain more than one road passing through them?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2009
+}

annotation_data/carcassonne_tier1/test_2010.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "What is the size of the minimum rectangle that can enclose all the tiles in the given landscape? Provide the answer in the format 'width x height'.",
+  "Answer": "3 x 2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2010
+}

annotation_data/carcassonne_tier1/test_2011.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many tiles have a building present on them in the given landscape?",
+  "Answer": "1",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2011
+}

annotation_data/carcassonne_tier1/test_2012.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "In the given landscape, how many tiles does the longest continuous road pass through?",
+  "Answer": "4",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2012
+}

annotation_data/carcassonne_tier1/test_2013.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many tiles are present in the given landscape?",
+  "Answer": "7",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2013
+}

annotation_data/carcassonne_tier1/test_2014.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the current landscape, how many tiles contain a single road?",
+  "Answer": "3",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2014
+}

annotation_data/carcassonne_tier1/test_2015.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many tiles contain grass but no road present on them in the given landscape?",
+  "Answer": "3",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2015
+}

annotation_data/carcassonne_tier1/test_2016.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many tiles contain a shield (or coat of arms) symbol in the given landscape?",
+  "Answer": "1",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2016
+}

annotation_data/carcassonne_tier1/test_2017.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Assuming that the goal is to get the current landscape to have a rectangular shape with no gaps, how many additional tiles are needed to be placed in order to achieve this?",
+  "Answer": "5",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2017
+}

annotation_data/carcassonne_tier1/test_2018.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many meeples are currently placed on the tiles in the given landscape?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2018
+}

annotation_data/carcassonne_tier1/test_2019.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Which color meeple is present on the tile containing only a road segment in the given landscape?",
+  "Answer": "red",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2019
+}

annotation_data/carcassonne_tier1/test_2020.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many tiles do not contain any meeples in the given landscape?",
+  "Answer": "4",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2020
+}

annotation_data/carcassonne_tier1/test_2021.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "From the given landscape, how many tiles contain sleeping meeples?",
+  "Answer": "1",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2021
+}

annotation_data/carcassonne_tier1/test_2022.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Assuming that the goal is to create a rectangular shape with no gaps, how many additional tiles are needed to be placed to achieve this?",
+  "Answer": "5",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2022
+}

annotation_data/carcassonne_tier1/test_2023.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Which color meeples is present on the tile containing at least one road segment in the given landscape?",
+  "Answer": "red, blue",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2023
+}

annotation_data/carcassonne_tier1/test_2024.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many standing yellow meeples are present in the given landscape?",
+  "Answer": "0",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2024
+}

annotation_data/carcassonne_tier1/test_2025.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the scoreboard, How many players are playing this game?",
+  "Answer": "3",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2025
+}

annotation_data/carcassonne_tier1/test_2026.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many points does the green player have according to the scoreboard?",
+  "Answer": "8",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2026
+}

annotation_data/carcassonne_tier1/test_2027.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many points does the blue player have according to the scoreboard?",
+  "Answer": "30",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_1.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2027
+}

annotation_data/carcassonne_tier1/test_2028.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many points does the blue player have according to the scoreboard?",
+  "Answer": "40",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2028
+}

annotation_data/carcassonne_tier1/test_2029.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many meeples are currently in the standing position on the scoreboard?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2029
+}

annotation_data/carcassonne_tier1/test_2030.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Assuming that meeples in any orientation (standing or sleeping) are considered to be same, how many points does the red player have according to the scoreboard?",
+  "Answer": "0",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2030
+}

annotation_data/carcassonne_tier1/test_2031.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Given the current scoreboard, how many players have their meeples in sleeping position?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2031
+}

annotation_data/carcassonne_tier1/test_2032.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Assuming that meeples in any orientation (standing or sleeping) are considered to be same, which player has the highest points according to the scoreboard?",
+  "Answer": "Blue",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2032
+}

annotation_data/carcassonne_tier1/test_2033.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Assuming that meeples in any orientation (standing or sleeping) are considered to be same, how many players have scored less than 10 points according to the scoreboard?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2033
+}

annotation_data/carcassonne_tier1/test_2034.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Assuming that meeples in any orientation (standing or sleeping) are considered to be same, what is the score of the green player according to the scoreboard?",
+  "Answer": "6",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2034
+}

annotation_data/carcassonne_tier1/test_2035.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "Assuming that meeples in any orientation (standing or sleeping) are considered to be same, which player has the second highest points according to the scoreboard?",
+  "Answer": "Green",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2035
+}

annotation_data/carcassonne_tier1/test_2036.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many tiles are present in the given gamestate image?",
+  "Answer": "31",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/farmers_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2036
+}

annotation_data/carcassonne_tier1/test_2037.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many standing meeples are present in the given gamestate image?",
+  "Answer": "1",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/farmers_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2037
+}

annotation_data/carcassonne_tier1/test_2038.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many meeples are present in the given gamestate image? Consider that both standing and lying meeples are counted as meeples.",
+  "Answer": "4",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/farmers_2.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2038
+}

annotation_data/carcassonne_tier1/test_2039.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "How many red meeples are present in the given gamestate image? Consider that both standing and lying meeples are counted as meeples.",
+  "Answer": "0",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/possible_tiles_3.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2039
+}

annotation_data/carcassonne_tier1/test_2040.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding. Counting the number of dominoes in the player's kingdom.",
+  "Difficulty Notes": "Very easy counting",
+  "Question": "In the given gamestate image, how many standing red meeples are present?",
+  "Answer": "0",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/possible_tiles_4.png",
+  "Game State Difficulty": 1,
+  "Rules Difficulty": 0,
+  "Strategy Difficulty": 0,
+  "tier": 1,
+  "json_game_state_url": "",
+  "ID": 2040
+}

annotation_data/carcassonne_tier2/test_2101.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding and rule understanding.",
+  "Difficulty Notes": "simple rule application",
+  "Question": "How many completed cities are there on the given landscape?",
+  "Answer": "1",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 2,
+  "Rules Difficulty": 2,
+  "Strategy Difficulty": 0,
+  "tier": 2,
+  "json_game_state_url": "",
+  "ID": 2101
+}

annotation_data/carcassonne_tier2/test_2102.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding and rule understanding.",
+  "Difficulty Notes": "simple rule application",
+  "Question": "How many completed roads are present in the given landscape?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 2,
+  "Rules Difficulty": 2,
+  "Strategy Difficulty": 0,
+  "tier": 2,
+  "json_game_state_url": "",
+  "ID": 2102
+}

annotation_data/carcassonne_tier2/test_2103.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding and rule understanding.",
+  "Difficulty Notes": "simple rule application",
+  "Question": "How many tiles are surrounding the monastery in the given landscape?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/features_1.png",
+  "Game State Difficulty": 2,
+  "Rules Difficulty": 2,
+  "Strategy Difficulty": 0,
+  "tier": 2,
+  "json_game_state_url": "",
+  "ID": 2103
+}

annotation_data/carcassonne_tier2/test_2104.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding and rule understanding.",
+  "Difficulty Notes": "simple rule application",
+  "Question": "Considering that the placement of the tile and meeple in this turn is completed, during scoring of this turn, how many points will the blue player earn?",
+  "Answer": "10",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_2.png",
+  "Game State Difficulty": 2,
+  "Rules Difficulty": 2,
+  "Strategy Difficulty": 0,
+  "tier": 2,
+  "json_game_state_url": "",
+  "ID": 2104
+}

annotation_data/carcassonne_tier2/test_2105.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding and rule understanding.",
+  "Difficulty Notes": "simple rule application",
+  "Question": "Considering that the placement of the tile and meeple in this turn is completed, during scoring of this turn, how many points will the yellow player earn? (Assume game does not end after this turn)",
+  "Answer": "0",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/meeple_landscape_2.png",
+  "Game State Difficulty": 2,
+  "Rules Difficulty": 2,
+  "Strategy Difficulty": 0,
+  "tier": 2,
+  "json_game_state_url": "",
+  "ID": 2105
+}

annotation_data/carcassonne_tier2/test_2106.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding and rule understanding.",
+  "Difficulty Notes": "simple rule application",
+  "Question": "According to the given scoreboard, which player has the highest score at this point in the game?",
+  "Answer": "green",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_3.png",
+  "Game State Difficulty": 2,
+  "Rules Difficulty": 2,
+  "Strategy Difficulty": 0,
+  "tier": 2,
+  "json_game_state_url": "",
+  "ID": 2106
+}

annotation_data/carcassonne_tier2/test_2107.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding and rule understanding.",
+  "Difficulty Notes": "simple rule application",
+  "Question": "How many players have a score of more than 25 points according to the given scoreboard?",
+  "Answer": "2",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/players_2.png",
+  "Game State Difficulty": 2,
+  "Rules Difficulty": 2,
+  "Strategy Difficulty": 0,
+  "tier": 2,
+  "json_game_state_url": "",
+  "ID": 2107
+}

annotation_data/carcassonne_tier2/test_2108.json

@@ -0,0 +1,15 @@
+{
+  "Game": "carcassonne",
+  "Test Description": "Testing basic game understanding and rule understanding.",
+  "Difficulty Notes": "simple rule application",
+  "Question": "Given the landscape, what are the possible tiles a player can place in the position marked with an `X`? Output the valid tiles in alphabetical order separated by commas.",
+  "Answer": "A,C",
+  "Rationale": "",
+  "game_state_url": "gamestates/images/carcassonne/possible_tiles_1.png",
+  "Game State Difficulty": 2,
+  "Rules Difficulty": 2,
+  "Strategy Difficulty": 0,
+  "tier": 2,
+  "json_game_state_url": "",
+  "ID": 2108
+}