Skip to content

Commit 48876a6

Browse files
roblourensseeM
roblourens
authored and
seeM
committed
Delete LanguageModelExtraDataPart (#248180)
* Delete LanguageModelExtraDataPart #246980 * Clean up helper types * Plumb non-image data through
1 parent faa1489 commit 48876a6

File tree

7 files changed

+60
-88
lines changed

7 files changed

+60
-88
lines changed

‎src/vs/base/common/marshallingIds.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,4 @@ export const enum MarshalledId {
2727
LanguageModelTextPart,
2828
LanguageModelPromptTsxPart,
2929
LanguageModelDataPart,
30-
LanguageModelExtraDataPart,
3130
}

‎src/vs/platform/extensions/common/extensionsApiProposals.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ const _allApiProposals = {
231231
},
232232
languageModelDataPart: {
233233
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts',
234-
version: 2
234+
version: 3
235235
},
236236
languageModelSystem: {
237237
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageModelSystem.d.ts',

‎src/vs/workbench/api/common/extHost.api.impl.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1833,7 +1833,6 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
18331833
LanguageModelToolResult: extHostTypes.LanguageModelToolResult,
18341834
LanguageModelToolResult2: extHostTypes.LanguageModelToolResult2,
18351835
LanguageModelDataPart: extHostTypes.LanguageModelDataPart,
1836-
LanguageModelExtraDataPart: extHostTypes.LanguageModelExtraDataPart,
18371836
ExtendedLanguageModelToolResult: extHostTypes.ExtendedLanguageModelToolResult,
18381837
PreparedTerminalToolInvocation: extHostTypes.PreparedTerminalToolInvocation,
18391838
LanguageModelChatToolMode: extHostTypes.LanguageModelChatToolMode,

‎src/vs/workbench/api/common/extHostTypeConverters.ts

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ import { IChatRequestVariableEntry, isImageVariableEntry } from '../../contrib/c
4545
import { IChatAgentMarkdownContentWithVulnerability, IChatCodeCitation, IChatCommandButton, IChatConfirmation, IChatContentInlineReference, IChatContentReference, IChatExtensionsContent, IChatFollowup, IChatMarkdownContent, IChatMoveMessage, IChatProgressMessage, IChatResponseCodeblockUriPart, IChatTaskDto, IChatTaskResult, IChatTextEdit, IChatTreeData, IChatUserActionEvent, IChatWarningMessage } from '../../contrib/chat/common/chatService.js';
4646
import { IToolData, IToolResult } from '../../contrib/chat/common/languageModelToolsService.js';
4747
import * as chatProvider from '../../contrib/chat/common/languageModels.js';
48-
import { IChatResponseDataPart, IChatResponsePromptTsxPart, IChatResponseTextPart } from '../../contrib/chat/common/languageModels.js';
48+
import { IChatMessageDataPart, IChatResponseDataPart, IChatResponsePromptTsxPart, IChatResponseTextPart } from '../../contrib/chat/common/languageModels.js';
4949
import { DebugTreeItemCollapsibleState, IDebugVisualizationTreeItem } from '../../contrib/debug/common/debug.js';
5050
import * as notebooks from '../../contrib/notebook/common/notebookCommon.js';
5151
import { CellEditType } from '../../contrib/notebook/common/notebookCommon.js';
@@ -2330,12 +2330,14 @@ export namespace LanguageModelChatMessage {
23302330
}
23312331
});
23322332
return new types.LanguageModelToolResultPart(c.toolCallId, content, c.isError);
2333-
} else if (c.type === 'image_url' || c.type === 'extra_data') {
2333+
} else if (c.type === 'image_url') {
23342334
// Non-stable types
23352335
return undefined;
2336-
} else {
2336+
} else if (c.type === 'tool_use') {
23372337
return new types.LanguageModelToolCallPart(c.toolCallId, c.name, c.parameters);
23382338
}
2339+
2340+
return undefined;
23392341
}).filter(c => c !== undefined);
23402342

23412343
const role = LanguageModelChatMessageRole.to(message.role);
@@ -2427,8 +2429,8 @@ export namespace LanguageModelChatMessage2 {
24272429
return new types.LanguageModelToolResultPart2(c.toolCallId, content, c.isError);
24282430
} else if (c.type === 'image_url') {
24292431
return new types.LanguageModelDataPart(c.value.data.buffer, c.value.mimeType);
2430-
} else if (c.type === 'extra_data') {
2431-
return new types.LanguageModelExtraDataPart(c.kind, c.data);
2432+
} else if (c.type === 'data') {
2433+
return new types.LanguageModelDataPart(c.data.buffer, c.mimeType);
24322434
} else {
24332435
return new types.LanguageModelToolCallPart(c.toolCallId, c.name, c.parameters);
24342436
}
@@ -2480,15 +2482,23 @@ export namespace LanguageModelChatMessage2 {
24802482
isError: c.isError
24812483
};
24822484
} else if (c instanceof types.LanguageModelDataPart) {
2483-
const value: chatProvider.IChatImageURLPart = {
2484-
mimeType: c.mimeType as chatProvider.ChatImageMimeType,
2485-
data: VSBuffer.wrap(c.data),
2486-
};
2485+
if (isImageDataPart(c)) {
2486+
const value: chatProvider.IChatImageURLPart = {
2487+
mimeType: c.mimeType as chatProvider.ChatImageMimeType,
2488+
data: VSBuffer.wrap(c.data),
2489+
};
24872490

2488-
return {
2489-
type: 'image_url',
2490-
value: value
2491-
};
2491+
return {
2492+
type: 'image_url',
2493+
value: value
2494+
};
2495+
} else {
2496+
return {
2497+
type: 'data',
2498+
mimeType: c.mimeType,
2499+
data: VSBuffer.wrap(c.data),
2500+
} satisfies IChatMessageDataPart;
2501+
}
24922502
} else if (c instanceof types.LanguageModelToolCallPart) {
24932503
return {
24942504
type: 'tool_use',
@@ -2501,12 +2511,6 @@ export namespace LanguageModelChatMessage2 {
25012511
type: 'text',
25022512
value: c.value
25032513
};
2504-
} else if (c instanceof types.LanguageModelExtraDataPart) {
2505-
return {
2506-
type: 'extra_data',
2507-
kind: c.kind,
2508-
data: c.data
2509-
} satisfies chatProvider.IChatMessagePart;
25102514
} else {
25112515
if (typeof c !== 'string') {
25122516
throw new Error('Unexpected chat message content type llm 2');
@@ -2527,6 +2531,19 @@ export namespace LanguageModelChatMessage2 {
25272531
}
25282532
}
25292533

2534+
function isImageDataPart(part: types.LanguageModelDataPart): boolean {
2535+
switch (part.mimeType) {
2536+
case types.ChatImageMimeType.PNG:
2537+
case types.ChatImageMimeType.JPEG:
2538+
case types.ChatImageMimeType.GIF:
2539+
case types.ChatImageMimeType.WEBP:
2540+
case types.ChatImageMimeType.BMP:
2541+
return true;
2542+
default:
2543+
return false;
2544+
}
2545+
}
2546+
25302547
export namespace ChatResponseMarkdownPart {
25312548
export function from(part: vscode.ChatResponseMarkdownPart): Dto<IChatMarkdownContent> {
25322549
return {

‎src/vs/workbench/api/common/extHostTypes.ts

Lines changed: 11 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -5012,19 +5012,19 @@ export class LanguageModelChatMessage implements vscode.LanguageModelChatMessage
50125012

50135013
export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessage2 {
50145014

5015-
static User(content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string): LanguageModelChatMessage2 {
5015+
static User(content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string): LanguageModelChatMessage2 {
50165016
return new LanguageModelChatMessage2(LanguageModelChatMessageRole.User, content, name);
50175017
}
50185018

5019-
static Assistant(content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string): LanguageModelChatMessage2 {
5019+
static Assistant(content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string): LanguageModelChatMessage2 {
50205020
return new LanguageModelChatMessage2(LanguageModelChatMessageRole.Assistant, content, name);
50215021
}
50225022

50235023
role: vscode.LanguageModelChatMessageRole;
50245024

5025-
private _content: (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] = [];
5025+
private _content: (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[] = [];
50265026

5027-
set content(value: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[]) {
5027+
set content(value: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[]) {
50285028
if (typeof value === 'string') {
50295029
// we changed this and still support setting content with a string property. this keep the API runtime stable
50305030
// despite the breaking change in the type definition.
@@ -5034,7 +5034,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag
50345034
}
50355035
}
50365036

5037-
get content(): (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] {
5037+
get content(): (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[] {
50385038
return this._content;
50395039
}
50405040

@@ -5050,7 +5050,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag
50505050
}
50515051
}
50525052

5053-
get content2(): (string | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] | undefined {
5053+
get content2(): (string | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[] | undefined {
50545054
return this.content.map(part => {
50555055
if (part instanceof LanguageModelTextPart) {
50565056
return part.value;
@@ -5061,7 +5061,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag
50615061

50625062
name: string | undefined;
50635063

5064-
constructor(role: vscode.LanguageModelChatMessageRole, content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string) {
5064+
constructor(role: vscode.LanguageModelChatMessageRole, content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string) {
50655065
this.role = role;
50665066
this.content = content;
50675067
this.name = name;
@@ -5110,13 +5110,13 @@ export class LanguageModelDataPart implements vscode.LanguageModelDataPart {
51105110
return new LanguageModelDataPart(data, mimeType as string);
51115111
}
51125112

5113-
static json(value: object): vscode.LanguageModelDataPart {
5113+
static json(value: object, mime: string = 'text/x-json'): vscode.LanguageModelDataPart {
51145114
const rawStr = JSON.stringify(value, undefined, '\t');
5115-
return new LanguageModelDataPart(VSBuffer.fromString(rawStr).buffer, 'json');
5115+
return new LanguageModelDataPart(VSBuffer.fromString(rawStr).buffer, mime);
51165116
}
51175117

5118-
static text(value: string): vscode.LanguageModelDataPart {
5119-
return new LanguageModelDataPart(VSBuffer.fromString(value).buffer, 'text/plain');
5118+
static text(value: string, mime: string = Mimes.text): vscode.LanguageModelDataPart {
5119+
return new LanguageModelDataPart(VSBuffer.fromString(value).buffer, mime);
51205120
}
51215121

51225122
toJSON() {
@@ -5136,24 +5136,6 @@ export enum ChatImageMimeType {
51365136
BMP = 'image/bmp',
51375137
}
51385138

5139-
export class LanguageModelExtraDataPart implements vscode.LanguageModelExtraDataPart {
5140-
kind: string;
5141-
data: any;
5142-
5143-
constructor(kind: string, data: any) {
5144-
this.kind = kind;
5145-
this.data = data;
5146-
}
5147-
5148-
toJSON() {
5149-
return {
5150-
$mid: MarshalledId.LanguageModelExtraDataPart,
5151-
kind: this.kind,
5152-
data: this.data,
5153-
};
5154-
}
5155-
}
5156-
51575139

51585140
export class LanguageModelPromptTsxPart {
51595141
value: unknown;

‎src/vs/workbench/contrib/chat/common/languageModels.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ export interface IChatMessageImagePart {
4040
value: IChatImageURLPart;
4141
}
4242

43-
export interface IChatMessageExtraDataPart {
44-
type: 'extra_data';
45-
kind: string;
46-
data: any;
43+
export interface IChatMessageDataPart {
44+
type: 'data';
45+
mimeType: string;
46+
data: VSBuffer;
4747
}
4848

4949
export interface IChatImageURLPart {
@@ -85,7 +85,7 @@ export interface IChatMessageToolResultPart {
8585
isError?: boolean;
8686
}
8787

88-
export type IChatMessagePart = IChatMessageTextPart | IChatMessageToolResultPart | IChatResponseToolUsePart | IChatMessageImagePart | IChatMessageExtraDataPart;
88+
export type IChatMessagePart = IChatMessageTextPart | IChatMessageToolResultPart | IChatResponseToolUsePart | IChatMessageImagePart | IChatMessageDataPart;
8989

9090
export interface IChatMessage {
9191
readonly name?: string | undefined;

‎src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
// version: 2
6+
// version: 3
77

88
declare module 'vscode' {
99

@@ -23,15 +23,15 @@ declare module 'vscode' {
2323
* @param content The content of the message.
2424
* @param name The optional name of a user for the message.
2525
*/
26-
static User(content: string | Array<LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelDataPart | LanguageModelExtraDataPart>, name?: string): LanguageModelChatMessage2;
26+
static User(content: string | Array<LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelDataPart>, name?: string): LanguageModelChatMessage2;
2727

2828
/**
2929
* Utility to create a new assistant message.
3030
*
3131
* @param content The content of the message.
3232
* @param name The optional name of a user for the message.
3333
*/
34-
static Assistant(content: string | Array<LanguageModelTextPart | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart>, name?: string): LanguageModelChatMessage2;
34+
static Assistant(content: string | Array<LanguageModelTextPart | LanguageModelToolCallPart | LanguageModelDataPart>, name?: string): LanguageModelChatMessage2;
3535

3636
/**
3737
* The role of this message.
@@ -42,7 +42,7 @@ declare module 'vscode' {
4242
* A string or heterogeneous array of things that a message can contain as content. Some parts may be message-type
4343
* specific for some models.
4444
*/
45-
content: Array<LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart>;
45+
content: Array<LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart>;
4646

4747
/**
4848
* The optional name of a user for this message.
@@ -56,7 +56,7 @@ declare module 'vscode' {
5656
* @param content The content of the message.
5757
* @param name The optional name of a user for the message.
5858
*/
59-
constructor(role: LanguageModelChatMessageRole, content: string | Array<LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart>, name?: string);
59+
constructor(role: LanguageModelChatMessageRole, content: string | Array<LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart>, name?: string);
6060
}
6161

6262
/**
@@ -70,9 +70,9 @@ declare module 'vscode' {
7070
*/
7171
static image(data: Uint8Array, mimeType: ChatImageMimeType): LanguageModelDataPart;
7272

73-
static json(value: object): LanguageModelDataPart;
73+
static json(value: any, mime?: string): LanguageModelDataPart;
7474

75-
static text(value: string): LanguageModelDataPart;
75+
static text(value: string, mime?: string): LanguageModelDataPart;
7676

7777
/**
7878
* The mime type which determines how the data property is interpreted.
@@ -102,31 +102,6 @@ declare module 'vscode' {
102102
BMP = 'image/bmp',
103103
}
104104

105-
/**
106-
* Tagging onto this proposal, because otherwise managing two different extensions of LanguageModelChatMessage could be confusing.
107-
* A language model response part containing arbitrary model-specific data, returned from a {@link LanguageModelChatResponse}.
108-
* TODO@API naming, looking at LanguageModelChatRequestOptions.modelOptions, but LanguageModelModelData is not very good.
109-
* LanguageModelOpaqueData from prompt-tsx?
110-
*/
111-
export class LanguageModelExtraDataPart {
112-
/**
113-
* The type of data. The allowed values and data types here are model-specific.
114-
*/
115-
kind: string;
116-
117-
/**
118-
* Extra model-specific data.
119-
*/
120-
data: any;
121-
122-
/**
123-
* Construct an extra data part with the given content.
124-
* @param value The image content of the part.
125-
*/
126-
constructor(kind: string, data: any);
127-
}
128-
129-
130105
/**
131106
* The result of a tool call. This is the counterpart of a {@link LanguageModelToolCallPart tool call} and
132107
* it can only be included in the content of a User message

0 commit comments

Comments
 (0)