Updated app.py

app.py

@@ -1,3 +1,4 @@
+# app.py
 import spaces
 import ast
 import torch
@@ -7,7 +8,7 @@ import base64
 from io import BytesIO
 
 from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
-from qwen_vl_utils import process_vision_info
+from qwen_vl_utils import process_vision_info  # include this file in your repo if not pip-installable
 
 _MODEL = Qwen2_5_VLForConditionalGeneration.from_pretrained(
     "ByteDance-Seed/UI-TARS-1.5-7B",
@@ -17,7 +18,7 @@ _MODEL = Qwen2_5_VLForConditionalGeneration.from_pretrained(
 
 _PROCESSOR = AutoProcessor.from_pretrained(
     "ByteDance-Seed/UI-TARS-1.5-7B",
-    size={"shortest_edge": 100 * 28 * 28, "longest_edge": 16384 * 28 * 28},
+    size={"shortest_edge": 100 * 28 * 28, "longest_edge": 16384 * 28 * 28},  # sane res
     use_fast=True,
 )
 
@@ -38,44 +39,32 @@ def draw_point(image: Image.Image, point=None, radius: int = 5):
 
 @spaces.GPU
 def navigate(screenshot, task: str):
-    """Run one inference step on the GUI‑reasoning model."""
-    messages = []
+    """Run one inference step on the GUI‑reasoning model.
+
+    Args:
+        screenshot (PIL.Image): Latest UI screenshot.
+        task (str): Natural‑language task description
+        history (list | str | None): Previous messages list. Accepts either an
+            actual Python list (via gr.JSON) or a JSON/Python‑literal string.
+    """
+
+
+    # ───────────────────── normalise history input ──────────────────────────
+
+    messages=[]
 
     prompt_header = (
-        "🤖 **GUI Agent Instructions**\n\n"
-        "You're an intelligent agent solving UI tasks through:\n"
-        "1. Visual understanding of screenshots\n"
-        "2. Natural language task interpretation\n"
-        "3. Action sequence generation\n\n"
-        "## Action Reference\n"
-        "```
-        "| Action Type       | Syntax                  |\n"
-        "|--------------------|-------------------------|\n"
-        "| Click Button       | click(start_box='(x,y)') |\n"
-        "| Double-Click       | left_double(start_box='(x,y)') |\n"
-        "| Drag Element       | drag(start_box='(x1,y1)', end_box='(x2,y2)') |\n"
-        "| Hotkey Input       | hotkey(key='Ctrl+A')    |\n"
-        "| Text Input         | type(content='Search term') |\n"
-        "| Scroll Action      | scroll(start_box='(x,y)', direction='down') |\n"
-        "| Wait & Recheck     | wait()                  |\n"
-        "| Task Completion    | finished(content='Result') |\n"
-        "```"
-        "\n**Note:**\n"
-        "1. Use 'win' key instead of 'meta' in hotkey commands\n"
-        "2. Include position coordinates in all spatial actions\n"
-        "3. Keep 'Thought' concise - max 3 sentence strategy\n"
-        f"\n**Task:**\n{task}"
-    )
+            "You are a GUI agent. You are given a task and your action history, with screenshots."
+            "You need to perform the next action to complete the task. \n\n## Output Format\n```\nThought: ...\nAction: ...\n```\n\n## Action Space\n\nclick(start_box='<|box_start|>(x1, y1)<|box_end|>')\nleft_double(start_box='<|box_start|>(x1, y1)<|box_end|>')\nright_single(start_box='<|box_start|<(x1, y1)>|box_end|>')\ndrag(start_box='<|box_start|>(x1, y1)<|box_end|>', end_box='<|box_start|>(x3, y3)<|box_end|>')\nhotkey(key='')\ntype(content='') #If you want to submit your input, use \"\\n\" at the end of `content`.\nscroll(start_box='<|box_start|>(x1, y1)<|box_end|>', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use English in `Thought` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part. Always use 'win' instead of 'meta' key\n\n"
+            f"## User Instruction\n{task}"
+        )
+    current = {"role":"user","content":[{"type":"text","text":prompt_header},{"type": "image_url", "image_url":screenshot}]}
 
-    current = {
-        "role": "user",
-        "content": [
-            {"type": "text", "text": prompt_header},
-            {"type": "image_url", "image_url": screenshot}
-        ]
-    }
     messages.append(current)
 
+    #New Comment 1
+    # ─────────────────────────── model forward ─────────────────────────────
+
     images, videos = process_vision_info(messages)
     text = processor.apply_chat_template(
         messages, tokenize=False, add_generation_prompt=True
@@ -90,43 +79,48 @@ def navigate(screenshot, task: str):
 
     generated = model.generate(**inputs, max_new_tokens=128)
     trimmed = [
-        out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated)
+        out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated)
     ]
     raw_out = processor.batch_decode(
         trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
     )[0]
 
+    # ─────── draw predicted click for quick visual verification (optional) ──────
     try:
         actions = ast.literal_eval(raw_out)
-        for act in (actions if isinstance(actions, list) else [actions]):
+        for act in actions if isinstance(actions, list) else [actions]:
             pos = act.get("position")
             if pos and isinstance(pos, list) and len(pos) == 2:
                 screenshot = draw_point(screenshot, pos)
     except Exception:
+        # decoding failed → just return original screenshot
         pass
 
     return screenshot, raw_out, messages
 
+# ────────────────────────── Gradio interface ───────────────────────────────
 
-demo = gr.Blocks()
-with gr.Row():
-    gr.Image(type="pil", label="🖼️ Screenshot Input").style(width=400)
-    gr.Textbox(
-        lines=1,
-        placeholder="e.g., 'Book a flight to Paris'",
-        label="🔍 Task Description"
-    ).style(width=400)
-
-with gr.Row():
-    gr.Image(label="📍 Click Point Visualization").style(width=400)
-    gr.Textbox(label="📝 Action Response").style(width=400)
-
-with gr.Row():
-    gr.JSON(label="📜 Conversation History").style(width=800)
+demo = gr.Interface(
+    fn=navigate,
+    inputs=[
+        gr.Image(type="pil", label="Screenshot"),
+        gr.Textbox(
+            lines=1,
+            placeholder="e.g. Search the weather for New York",
+            label="Task",
+        )
+    ],
+    outputs=[
+        gr.Image(label="With Click Point"),
+        gr.Textbox(label="Raw Action JSON"),
+        gr.JSON(label="Updated Conversation History")
+    ],
+    title="UI-Tars Navigation Demo",
+)
 
 demo.launch(
     server_name="0.0.0.0",
     server_port=7860,
-    share=False,
-    ssr_mode=False
+    share=False,      # or True if you need a public link
+    ssr_mode=False,   # turn off experimental SSR so the process blocks
 )