fix(agent): abort not break immediately (#49)

* chore: call_user action

* fix(agent): abort not break immediately

* fix(ui): disable logo dragged

Author: ycjcl868

packages/shared/package.json

@@ -17,10 +17,10 @@
   "typesVersions": {
     "*": {
       "types": [
-        "./dist/types/index.d.ts"
+        "./src/types/index.ts"
       ],
       "*": [
-        "./dist/*"
+        "./src/*"
       ]
     }
   },
@@ -38,7 +38,8 @@
     "registry": "https://registry.npmjs.org/"
   },
   "files": [
-    "dist"
+    "dist",
+    "src"
   ],
   "dependencies": {},
   "devDependencies": {

src/main/agent/execute.ts

@@ -245,7 +245,7 @@ export const execute = async (executeParams: ExecuteParams) => {
       break;
     }
 
-    case 'screenshot':
+    case 'call_user':
     case 'finished':
       break;
 

src/main/agent/index.ts

@@ -226,7 +226,9 @@ export class ComputerUseAgent {
         logger.info('[vlmParams_conversations]:', vlmParams.conversations);
         logger.info('[vlmParams_images_len]:', vlmParams.images.length);
 
-        const vlmRes = await vlm.invoke(vlmParams);
+        const vlmRes = await vlm.invoke(vlmParams, {
+          abortController,
+        });
 
         if (!vlmRes?.prediction) {
           continue;

src/main/agent/llm/base.ts

@@ -9,6 +9,10 @@ export interface VlmRequest {
   images: string[];
 }
 
+export interface VlmRequestOptions {
+  abortController?: AbortController | null;
+}
+
 export interface VlmResponse {
   prediction: string;
   reflections?: string[];
@@ -32,5 +36,8 @@ export abstract class VLM<
   K extends VlmResponse = VlmResponse,
 > {
   abstract get vlmModel(): string;
-  abstract invoke({ conversations, images }: T): Promise<K>;
+  abstract invoke(
+    { conversations, images }: T,
+    options?: VlmRequestOptions,
+  ): Promise<K>;
 }

src/main/agent/llm/ui-tars.ts

@@ -10,7 +10,7 @@ import { store } from '@main/store/create';
 import { preprocessResizeImage } from '@main/utils/image';
 
 import { MAX_PIXELS } from '../constant';
-import { VLM, VlmRequest, VlmResponse } from './base';
+import { VLM, VlmRequest, VlmRequestOptions, VlmResponse } from './base';
 
 export interface UITARSOptions {
   reflection: boolean;
@@ -31,7 +31,11 @@ export class UITARS implements VLM<VlmRequest, VlmResponse> {
 
   // [image, prompt]
   // [gpt, image]
-  async invoke({ conversations, images }: VlmRequest) {
+  async invoke(
+    { conversations, images }: VlmRequest,
+    options?: VlmRequestOptions,
+  ) {
+    const { abortController } = options ?? {};
     const compressedImages = await Promise.all(
       images.map((image) => preprocessResizeImage(image, MAX_PIXELS)),
     );
@@ -51,19 +55,24 @@ export class UITARS implements VLM<VlmRequest, VlmResponse> {
 
     const startTime = Date.now();
     const result = await openai.chat.completions
-      .create({
-        model: this.vlmModel,
-        max_tokens: 1000,
-        stream: false,
-        temperature: 0,
-        top_p: 0.7,
-        seed: null,
-        stop: null,
-        frequency_penalty: null,
-        presence_penalty: null,
-        // messages
-        messages,
-      })
+      .create(
+        {
+          model: this.vlmModel,
+          max_tokens: 1000,
+          stream: false,
+          temperature: 0,
+          top_p: 0.7,
+          seed: null,
+          stop: null,
+          frequency_penalty: null,
+          presence_penalty: null,
+          // messages
+          messages,
+        },
+        {
+          signal: abortController?.signal,
+        },
+      )
       .finally(() => {
         logger.info(`[vlm_invoke_time_cost]: ${Date.now() - startTime}ms`);
       });

src/main/store/create.ts

@@ -81,6 +81,10 @@ export const store = createStore<AppState>(
       },
 
       RUN_AGENT: async () => {
+        if (get().thinking) {
+          return;
+        }
+
         set({ abortController: new AbortController(), thinking: true });
 
         await runAgent(set, get);
@@ -91,6 +95,7 @@ export const store = createStore<AppState>(
         set({ status: StatusEnum.END, thinking: false });
         showWindow();
         get().abortController?.abort();
+
         closeScreenMarker();
       },
       SET_INSTRUCTIONS: (instructions) => {

src/renderer/src/components/Header/index.tsx

@@ -15,7 +15,12 @@ export default function Header({ className }: { className?: string }) {
     <Box position="relative" textAlign="center" className={className}>
       <Flex alignItems="center" justifyContent="center">
         <HStack>
-          <Image alt="UI-TARS Logo" src={logoVector} h="40px" />
+          <Image
+            alt="UI-TARS Logo"
+            src={logoVector}
+            h="40px"
+            draggable={false}
+          />
         </HStack>
         <Box position="absolute" right="4">
           <IconButton