| 
															
																@@ -62,7 +62,7 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   }, 
															 | 
															
															 | 
															
																   }, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   { 
															 | 
															
															 | 
															
																   { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "cell_type": "code", 
															 | 
															
															 | 
															
																    "cell_type": "code", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-   "execution_count": 3, 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+   "execution_count": 2, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "metadata": {}, 
															 | 
															
															 | 
															
																    "metadata": {}, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "outputs": [], 
															 | 
															
															 | 
															
																    "outputs": [], 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "source": [ 
															 | 
															
															 | 
															
																    "source": [ 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -87,7 +87,7 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   }, 
															 | 
															
															 | 
															
																   }, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   { 
															 | 
															
															 | 
															
																   { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "cell_type": "code", 
															 | 
															
															 | 
															
																    "cell_type": "code", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-   "execution_count": 4, 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+   "execution_count": 3, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "metadata": {}, 
															 | 
															
															 | 
															
																    "metadata": {}, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "outputs": [], 
															 | 
															
															 | 
															
																    "outputs": [], 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "source": [ 
															 | 
															
															 | 
															
																    "source": [ 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -142,7 +142,7 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   }, 
															 | 
															
															 | 
															
																   }, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   { 
															 | 
															
															 | 
															
																   { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "cell_type": "code", 
															 | 
															
															 | 
															
																    "cell_type": "code", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-   "execution_count": 7, 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+   "execution_count": 5, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "metadata": {}, 
															 | 
															
															 | 
															
																    "metadata": {}, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "outputs": [], 
															 | 
															
															 | 
															
																    "outputs": [], 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "source": [ 
															 | 
															
															 | 
															
																    "source": [ 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -194,7 +194,7 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   }, 
															 | 
															
															 | 
															
																   }, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   { 
															 | 
															
															 | 
															
																   { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "cell_type": "code", 
															 | 
															
															 | 
															
																    "cell_type": "code", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-   "execution_count": 8, 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+   "execution_count": 6, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "metadata": {}, 
															 | 
															
															 | 
															
																    "metadata": {}, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "outputs": [], 
															 | 
															
															 | 
															
																    "outputs": [], 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "source": [ 
															 | 
															
															 | 
															
																    "source": [ 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -307,6 +307,8 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "\n", 
															 | 
															
															 | 
															
																     "\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "This section indicates that there is a textbox with a name \"Flight destination input\" filled with San Francisco (SFO). There is also a button with the name \"Swap departure airport and destination airport\". Another textbox with the name \"Flight destination input\" not filled with any text. There are also buttons with the names \"Start date\", \"End date\", which are not filled with any dates, and a button named \"Search\".\n", 
															 | 
															
															 | 
															
																     "This section indicates that there is a textbox with a name \"Flight destination input\" filled with San Francisco (SFO). There is also a button with the name \"Swap departure airport and destination airport\". Another textbox with the name \"Flight destination input\" not filled with any text. There are also buttons with the names \"Start date\", \"End date\", which are not filled with any dates, and a button named \"Search\".\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "\n", 
															 | 
															
															 | 
															
																     "\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    "Retry actions at most 2 times before trying a different action.\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    "\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "### Examples:\n", 
															 | 
															
															 | 
															
																     "### Examples:\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "1. To click on a button labeled \"Search\":\n", 
															 | 
															
															 | 
															
																     "1. To click on a button labeled \"Search\":\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "   {\n", 
															 | 
															
															 | 
															
																     "   {\n", 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -353,7 +355,7 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   }, 
															 | 
															
															 | 
															
																   }, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   { 
															 | 
															
															 | 
															
																   { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "cell_type": "code", 
															 | 
															
															 | 
															
																    "cell_type": "code", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-   "execution_count": 9, 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+   "execution_count": 7, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "metadata": {}, 
															 | 
															
															 | 
															
																    "metadata": {}, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "outputs": [], 
															 | 
															
															 | 
															
																    "outputs": [], 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "source": [ 
															 | 
															
															 | 
															
																    "source": [ 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -481,7 +483,7 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   }, 
															 | 
															
															 | 
															
																   }, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																   { 
															 | 
															
															 | 
															
																   { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "cell_type": "code", 
															 | 
															
															 | 
															
																    "cell_type": "code", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-   "execution_count": 10, 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+   "execution_count": 8, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "metadata": {}, 
															 | 
															
															 | 
															
																    "metadata": {}, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "outputs": [], 
															 | 
															
															 | 
															
																    "outputs": [], 
															 | 
														
													
												
													
														
															| 
															 | 
															
																    "source": [ 
															 | 
															
															 | 
															
																    "source": [ 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -566,13 +568,13 @@ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                    temperature=0.0,\n", 
															 | 
															
															 | 
															
																     "                    temperature=0.0,\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                    messages=[\n", 
															 | 
															
															 | 
															
																     "                    messages=[\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                        {\"role\": \"system\", \"content\": execution_prompt},\n", 
															 | 
															
															 | 
															
																     "                        {\"role\": \"system\", \"content\": execution_prompt},\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-    "                        {\"role\": \"system\", \"content\": f\"Few shot examples: {few_shot_examples}. Just a fewexamples, user will assign you VERY range set of tasks.\"},\n", 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    "                        {\"role\": \"system\", \"content\": f\"Few shot examples: {few_shot_examples}. Just a few examples, user will assign you VERY range set of tasks.\"},\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                        {\"role\": \"system\", \"content\": f\"Plan to execute: {steps}\\n\\n Accessibility Tree: {previous_context}\\n\\n, previous actions: {previous_actions}\"},\n", 
															 | 
															
															 | 
															
																     "                        {\"role\": \"system\", \"content\": f\"Plan to execute: {steps}\\n\\n Accessibility Tree: {previous_context}\\n\\n, previous actions: {previous_actions}\"},\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                        {\"role\": \"user\", \"content\": \n", 
															 | 
															
															 | 
															
																     "                        {\"role\": \"user\", \"content\": \n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                         [\n", 
															 | 
															
															 | 
															
																     "                         [\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                            {\n", 
															 | 
															
															 | 
															
																     "                            {\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                                \"type\": \"text\",\n", 
															 | 
															
															 | 
															
																     "                                \"type\": \"text\",\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-    "                                \"text\": f'What should be the next action to accomplish the task: {task} based on the previous actions and the current state? Remember to review the plan. Provide the next action in JSON format strictly as specified above.',\n", 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+    "                                \"text\": f'What should be the next action to accomplish the task: {task} based on the current state? Remember to review the plan and select the next action based on the current state. Provide the next action in JSON format strictly as specified above.',\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                            },\n", 
															 | 
															
															 | 
															
																     "                            },\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                            {\n", 
															 | 
															
															 | 
															
																     "                            {\n", 
															 | 
														
													
												
													
														
															| 
															 | 
															
																     "                                \"type\": \"image_url\",\n", 
															 | 
															
															 | 
															
																     "                                \"type\": \"image_url\",\n", 
															 |