data/gemini/aitw_mcq_dataset_absolute_single_step.json

{
  "12172380859428428757": {
    "episode_goal": "Open a new Chrome private window",
    "steps": [
      {
        "step_id": 0,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12172380859428428757_step_0.png",
        "action_history": "",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [382, 838]",
            "Type: 'ok'",
            "Tap: [291, 838]",
            "Swipe: Up"
          ],
          "correct_answer_index": 0
        }
      },
      {
        "step_id": 1,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12172380859428428757_step_1.png",
        "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [224, 776]",
            "Type: 'settings'",
            "Tap: [287, 814]",
            "Button: Press Home"
          ],
          "correct_answer_index": 0
        }
      },
      {
        "step_id": 2,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12172380859428428757_step_2.png",
        "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [156, 776]",
            "Type: 'search'",
            "Tap: [420, 775]",
            "Button: Press Enter"
          ],
          "correct_answer_index": 2
        }
      },
      {
        "step_id": 3,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12172380859428428757_step_3.png",
        "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [468, 64]",
            "Tap: [516, 64]",
            "Button: Press Back",
            "Swipe: Right"
          ],
          "correct_answer_index": 0
        }
      },
      {
        "step_id": 4,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12172380859428428757_step_4.png",
        "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [44, 78]",
            "Type: 'search'",
            "Tap: [150, 105]",
            "Button: Press Enter"
          ],
          "correct_answer_index": 0
        }
      },
      {
        "step_id": 5,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12172380859428428757_step_5.png",
        "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Swipe: Left",
            "Tap: [467, 62]",
            "Type: 'ok'",
            "Tap: [515, 62]"
          ],
          "correct_answer_index": 1
        }
      },
      {
        "step_id": 6,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12172380859428428757_step_6.png",
        "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]\nStep 5: Tapped at pixel coordinates (x,y): [467, 62]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Swipe: Left",
            "Tap: [514, 64]",
            "Button: Press Home",
            "Tap: [514, 116]"
          ],
          "correct_answer_index": 1
        }
      },
      {
        "step_id": 7,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12172380859428428757_step_7.png",
        "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]\nStep 5: Tapped at pixel coordinates (x,y): [467, 62]\nStep 6: Tapped at pixel coordinates (x,y): [514, 64]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [370, 127]",
            "Type: 'settings'",
            "Tap: [370, 78]",
            "Swipe: Up"
          ],
          "correct_answer_index": 0
        }
      }
    ]
  },
  "9027843537045096280": {
    "episode_goal": "Open a new Chrome window",
    "steps": [
      {
        "step_id": 1,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_9027843537045096280_step_1.png",
        "action_history": "",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Button: Press Back",
            "Swipe: Left",
            "Tap: [365, 826]",
            "Tap: [277, 826]"
          ],
          "correct_answer_index": 2
        }
      }
    ]
  },
  "2247237893045084693": {
    "episode_goal": "Open the calendar and show me this week's events?",
    "steps": [
      {
        "step_id": 1,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_2247237893045084693_step_1.png",
        "action_history": "",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [85, 320]",
            "Type: 'settings'",
            "Tap: [379, 183]",
            "Swipe: Left"
          ],
          "correct_answer_index": 2
        }
      },
      {
        "step_id": 2,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_2247237893045084693_step_2.png",
        "action_history": "Step 1: Tapped at pixel coordinates (x,y): [379, 183]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Button: Press Enter",
            "Tap: [499, 976]",
            "Swipe: Left",
            "Tap: [442, 976]"
          ],
          "correct_answer_index": 1
        }
      },
      {
        "step_id": 3,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_2247237893045084693_step_3.png",
        "action_history": "Step 1: Tapped at pixel coordinates (x,y): [379, 183]\nStep 2: Tapped at pixel coordinates (x,y): [499, 976]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [302, 950]",
            "Swipe: Down",
            "Button: Press Enter",
            "Tap: [270, 600]"
          ],
          "correct_answer_index": 0
        }
      }
    ]
  },
  "12224608131504749719": {
    "episode_goal": "How much does a 2x4x8 board cost at Lowes?",
    "steps": [
      {
        "step_id": 2,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12224608131504749719_step_2.png",
        "action_history": "",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [289, 950]",
            "Swipe: Left",
            "Tap: [387, 852]",
            "Button: Press Home"
          ],
          "correct_answer_index": 0
        }
      },
      {
        "step_id": 4,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_12224608131504749719_step_4.png",
        "action_history": "Step 2: Tapped at pixel coordinates (x,y): [289, 950]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [286, 129]",
            "Tap: [286, 193]",
            "Button: Press Home",
            "Type: 'settings'"
          ],
          "correct_answer_index": 0
        }
      }
    ]
  },
  "1307957808436696980": {
    "episode_goal": "Google the capital of Mexico",
    "steps": [
      {
        "step_id": 3,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_1307957808436696980_step_3.png",
        "action_history": "",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Button: Press Home",
            "Tap: [78, 570]",
            "Tap: [320, 974]",
            "Type: 'ok'"
          ],
          "correct_answer_index": 2
        }
      },
      {
        "step_id": 5,
        "screenshot_path": "/content/aitw_images_absolute_single_step/episode_1307957808436696980_step_5.png",
        "action_history": "Step 3: Tapped at pixel coordinates (x,y): [320, 974]",
        "ground_truth_format": "mcq",
        "questions": {
          "question": "Based on the goal and screen, what is the precise next action to perform?",
          "options": [
            "Tap: [185, 114]",
            "Type: 'ok'",
            "Swipe: Right",
            "Tap: [199, 193]"
          ],
          "correct_answer_index": 0
        }
      }
    ]
  }
}