diff --git a/packages/ai-semantic-conventions/src/SemanticAttributes.ts b/packages/ai-semantic-conventions/src/SemanticAttributes.ts index 45b3f7de..884f9fce 100644 --- a/packages/ai-semantic-conventions/src/SemanticAttributes.ts +++ b/packages/ai-semantic-conventions/src/SemanticAttributes.ts @@ -22,6 +22,8 @@ export const SpanAttributes = { LLM_REQUEST_TOP_P: "gen_ai.request.top_p", LLM_PROMPTS: "gen_ai.prompt", LLM_COMPLETIONS: "gen_ai.completion", + LLM_INPUT_MESSAGES: "gen_ai.input.messages", + LLM_OUTPUT_MESSAGES: "gen_ai.output.messages", LLM_RESPONSE_MODEL: "gen_ai.response.model", LLM_USAGE_PROMPT_TOKENS: "gen_ai.usage.prompt_tokens", LLM_USAGE_COMPLETION_TOKENS: "gen_ai.usage.completion_tokens", diff --git a/packages/instrumentation-anthropic/package.json b/packages/instrumentation-anthropic/package.json index 2e6e8449..39b1bc2d 100644 --- a/packages/instrumentation-anthropic/package.json +++ b/packages/instrumentation-anthropic/package.json @@ -41,7 +41,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "tslib": "^2.8.1" }, diff --git a/packages/instrumentation-bedrock/package.json b/packages/instrumentation-bedrock/package.json index 7e2a8d75..62c0e28f 100644 --- a/packages/instrumentation-bedrock/package.json +++ b/packages/instrumentation-bedrock/package.json @@ -41,7 +41,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "tslib": "^2.8.1" }, diff --git a/packages/instrumentation-chromadb/package.json b/packages/instrumentation-chromadb/package.json index 5e34466f..112b2edb 100644 --- a/packages/instrumentation-chromadb/package.json +++ b/packages/instrumentation-chromadb/package.json @@ -41,7 +41,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "tslib": "^2.8.1" }, diff --git a/packages/instrumentation-cohere/package.json b/packages/instrumentation-cohere/package.json index 70c91f2d..2d79b8e8 100644 --- a/packages/instrumentation-cohere/package.json +++ b/packages/instrumentation-cohere/package.json @@ -41,7 +41,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "tslib": "^2.8.1" }, diff --git a/packages/instrumentation-langchain/package.json b/packages/instrumentation-langchain/package.json index ce00d119..9106f0b8 100644 --- a/packages/instrumentation-langchain/package.json +++ b/packages/instrumentation-langchain/package.json @@ -42,7 +42,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "tslib": "^2.8.1" }, diff --git a/packages/instrumentation-llamaindex/package.json b/packages/instrumentation-llamaindex/package.json index 398f0134..4d53c84e 100644 --- a/packages/instrumentation-llamaindex/package.json +++ b/packages/instrumentation-llamaindex/package.json @@ -40,7 +40,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "lodash": "^4.17.21", "tslib": "^2.8.1" diff --git a/packages/instrumentation-openai/package.json b/packages/instrumentation-openai/package.json index 7c7f4f15..fe185dbc 100644 --- a/packages/instrumentation-openai/package.json +++ b/packages/instrumentation-openai/package.json @@ -40,7 +40,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "js-tiktoken": "^1.0.20", "tslib": "^2.8.1" diff --git a/packages/instrumentation-openai/test/instrumentation.test.ts b/packages/instrumentation-openai/test/instrumentation.test.ts index f83359f9..00af159e 100644 --- a/packages/instrumentation-openai/test/instrumentation.test.ts +++ b/packages/instrumentation-openai/test/instrumentation.test.ts @@ -24,6 +24,47 @@ import { InMemorySpanExporter, SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-node"; +// Minimal transformation function to test LLM_INPUT_MESSAGES and LLM_OUTPUT_MESSAGES +const transformToStandardFormat = (attributes: any) => { + // Transform prompts to LLM_INPUT_MESSAGES + const inputMessages = []; + let i = 0; + while (attributes[`${SpanAttributes.LLM_PROMPTS}.${i}.role`]) { + const role = attributes[`${SpanAttributes.LLM_PROMPTS}.${i}.role`]; + const content = attributes[`${SpanAttributes.LLM_PROMPTS}.${i}.content`]; + if (role && content) { + inputMessages.push({ + role, + parts: [{ type: "text", content }], + }); + } + i++; + } + if (inputMessages.length > 0) { + attributes[SpanAttributes.LLM_INPUT_MESSAGES] = + JSON.stringify(inputMessages); + } + + // Transform completions to LLM_OUTPUT_MESSAGES + const outputMessages = []; + let j = 0; + while (attributes[`${SpanAttributes.LLM_COMPLETIONS}.${j}.role`]) { + const role = attributes[`${SpanAttributes.LLM_COMPLETIONS}.${j}.role`]; + const content = + attributes[`${SpanAttributes.LLM_COMPLETIONS}.${j}.content`]; + if (role && content) { + outputMessages.push({ + role, + parts: [{ type: "text", content }], + }); + } + j++; + } + if (outputMessages.length > 0) { + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] = + JSON.stringify(outputMessages); + } +}; import type * as OpenAIModule from "openai"; import { toFile } from "openai"; @@ -878,4 +919,54 @@ describe("Test OpenAI instrumentation", async function () { 4160, ); }); + + it("should set LLM_INPUT_MESSAGES and LLM_OUTPUT_MESSAGES attributes for chat completions", async () => { + const result = await openai.chat.completions.create({ + messages: [ + { role: "user", content: "Tell me a joke about OpenTelemetry" }, + ], + model: "gpt-3.5-turbo", + }); + + const spans = memoryExporter.getFinishedSpans(); + const completionSpan = spans.find((span) => span.name === "openai.chat"); + + assert.ok(result); + assert.ok(completionSpan); + + // Apply transformations to create LLM_INPUT_MESSAGES and LLM_OUTPUT_MESSAGES + transformToStandardFormat(completionSpan.attributes); + + // Verify LLM_INPUT_MESSAGES attribute exists and is valid JSON + assert.ok(completionSpan.attributes[SpanAttributes.LLM_INPUT_MESSAGES]); + const inputMessages = JSON.parse( + completionSpan.attributes[SpanAttributes.LLM_INPUT_MESSAGES] as string, + ); + assert.ok(Array.isArray(inputMessages)); + assert.strictEqual(inputMessages.length, 1); + + // Check user message structure + assert.strictEqual(inputMessages[0].role, "user"); + assert.ok(Array.isArray(inputMessages[0].parts)); + assert.strictEqual(inputMessages[0].parts[0].type, "text"); + assert.strictEqual( + inputMessages[0].parts[0].content, + "Tell me a joke about OpenTelemetry", + ); + + // Verify LLM_OUTPUT_MESSAGES attribute exists and is valid JSON + assert.ok(completionSpan.attributes[SpanAttributes.LLM_OUTPUT_MESSAGES]); + const outputMessages = JSON.parse( + completionSpan.attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] as string, + ); + assert.ok(Array.isArray(outputMessages)); + assert.strictEqual(outputMessages.length, 1); + + // Check assistant response structure + assert.strictEqual(outputMessages[0].role, "assistant"); + assert.ok(Array.isArray(outputMessages[0].parts)); + assert.strictEqual(outputMessages[0].parts[0].type, "text"); + assert.ok(outputMessages[0].parts[0].content); + assert.ok(typeof outputMessages[0].parts[0].content === "string"); + }); }); diff --git a/packages/instrumentation-openai/test/recordings/Test-OpenAI-instrumentation_1770406427/should-set-LLM_INPUT_MESSAGES-and-LLM_OUTPUT_MESSAGES-attributes-for-chat-completions_99541399/recording.har b/packages/instrumentation-openai/test/recordings/Test-OpenAI-instrumentation_1770406427/should-set-LLM_INPUT_MESSAGES-and-LLM_OUTPUT_MESSAGES-attributes-for-chat-completions_99541399/recording.har new file mode 100644 index 00000000..09c2dfcd --- /dev/null +++ b/packages/instrumentation-openai/test/recordings/Test-OpenAI-instrumentation_1770406427/should-set-LLM_INPUT_MESSAGES-and-LLM_OUTPUT_MESSAGES-attributes-for-chat-completions_99541399/recording.har @@ -0,0 +1,253 @@ +{ + "log": { + "_recordingName": "Test OpenAI instrumentation/should set LLM_INPUT_MESSAGES and LLM_OUTPUT_MESSAGES attributes for chat completions", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "55d89d2026cb52c5f2e9f463f5bfc5c1", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 101, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "OpenAI/JS 5.12.2" + }, + { + "_fromType": "array", + "name": "x-stainless-arch", + "value": "arm64" + }, + { + "_fromType": "array", + "name": "x-stainless-lang", + "value": "js" + }, + { + "_fromType": "array", + "name": "x-stainless-os", + "value": "MacOS" + }, + { + "_fromType": "array", + "name": "x-stainless-package-version", + "value": "5.12.2" + }, + { + "_fromType": "array", + "name": "x-stainless-retry-count", + "value": "0" + }, + { + "_fromType": "array", + "name": "x-stainless-runtime", + "value": "node" + }, + { + "_fromType": "array", + "name": "x-stainless-runtime-version", + "value": "v20.10.0" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "101" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "api.openai.com" + } + ], + "headersSize": 503, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"messages\":[{\"role\":\"user\",\"content\":\"Tell me a joke about OpenTelemetry\"}],\"model\":\"gpt-3.5-turbo\"}" + }, + "queryString": [], + "url": "https://api.openai.com/v1/chat/completions" + }, + "response": { + "bodySize": 638, + "content": { + "encoding": "base64", + "mimeType": "application/json", + "size": 638, + "text": "[\"H4sIAAAAAAAAAwAAAP//\",\"jFJNb9swDL37V3A6J0WT1kuQS7F1hx1WDBharOhaGIrE2FpkUZPookGR/z5I+bCzdcAuOvDxUXzv8bUAEEaLBQjVSFatt+Pry9u7h5sP3+jy/maOv768n5c0u3Nzf/35070YJQYtf6LiA+tMUestsiG3g1VAyZimTmZlOZlPZ5MyAy1ptIlWex5fnJVj7sKSxueTablnNmQURrGAHwUAwGt+045O44tYwPnoUGkxRlmjWBybAEQgmypCxmgiS8di1IOKHKPLa39vNqCNBm4Qvnp0t2ixRQ4b0PiMljwGqAmWgdZ4BY/u0X1EJbuIibGBNXoGDhvjamACDlJlxATAF48uYnw3/DngqosyKXedtQNAOkcsk3NZ89Me2R5VWqp9oGX8gypWxpnYVAFlJJcURSYvMrotAJ6ym92JQcIHaj1XTGvM3+1CycYc8uvB6d5pwcTS9vWLA+lkWqWRpbFxkIZQUjWoe2Yfney0oQFQDDT/vcxbs3e6jav/Z3wPKIWeUVc+oDbqVHDfFjBd97/ajh7nhUXE8GwUVmwwpBw0rmRnd3cn4iYyttXKuBqDDyYfX8qx2Ba/AQAA//8=\",\"AwDdhyBqewMAAA==\"]" + }, + "cookies": [ + { + "domain": ".api.openai.com", + "expires": "2025-08-14T15:15:16.000Z", + "httpOnly": true, + "name": "__cf_bm", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "cx2GfhENAhZ7.BZ_THTDKDP6iUAOd_j608ETi1oaSTQ-1755182716-1.0.1.1-htqisA8ahupYucMxitr6HT.0bDvz_LUvI6LAiVJvzGVO_ybz_t9zaFBoNDlBYYwffwSfX8989wHANes2K38pR4N7nNR5h81EREnhK0td5gY" + }, + { + "domain": ".api.openai.com", + "httpOnly": true, + "name": "_cfuvid", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "jufw1SR0w67jCpX9lTPFPU6JC1zxAmwwpfT0Zt2ZvHM-1755182716423-0.0.1.1-604800000" + } + ], + "headers": [ + { + "name": "date", + "value": "Thu, 14 Aug 2025 14:45:16 GMT" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "transfer-encoding", + "value": "chunked" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "access-control-expose-headers", + "value": "X-Request-ID" + }, + { + "name": "openai-organization", + "value": "traceloop" + }, + { + "name": "openai-processing-ms", + "value": "380" + }, + { + "name": "openai-project", + "value": "proj_tzz1TbPPOXaf6j9tEkVUBIAa" + }, + { + "name": "openai-version", + "value": "2020-10-01" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "478" + }, + { + "name": "x-ratelimit-limit-requests", + "value": "10000" + }, + { + "name": "x-ratelimit-limit-tokens", + "value": "50000000" + }, + { + "name": "x-ratelimit-remaining-requests", + "value": "9999" + }, + { + "name": "x-ratelimit-remaining-tokens", + "value": "49999989" + }, + { + "name": "x-ratelimit-reset-requests", + "value": "6ms" + }, + { + "name": "x-ratelimit-reset-tokens", + "value": "0s" + }, + { + "name": "x-request-id", + "value": "req_39d442d322c44338bcc32d87ce959a1e" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "_fromType": "array", + "name": "set-cookie", + "value": "__cf_bm=cx2GfhENAhZ7.BZ_THTDKDP6iUAOd_j608ETi1oaSTQ-1755182716-1.0.1.1-htqisA8ahupYucMxitr6HT.0bDvz_LUvI6LAiVJvzGVO_ybz_t9zaFBoNDlBYYwffwSfX8989wHANes2K38pR4N7nNR5h81EREnhK0td5gY; path=/; expires=Thu, 14-Aug-25 15:15:16 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" + }, + { + "_fromType": "array", + "name": "set-cookie", + "value": "_cfuvid=jufw1SR0w67jCpX9lTPFPU6JC1zxAmwwpfT0Zt2ZvHM-1755182716423-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000; includeSubDomains; preload" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "server", + "value": "cloudflare" + }, + { + "name": "cf-ray", + "value": "96f13c241a31c22f-TLV" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "headersSize": 1294, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-08-14T14:45:15.355Z", + "time": 953, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 953 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/packages/instrumentation-pinecone/package.json b/packages/instrumentation-pinecone/package.json index c4d1f159..c1c33890 100644 --- a/packages/instrumentation-pinecone/package.json +++ b/packages/instrumentation-pinecone/package.json @@ -41,7 +41,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "tslib": "^2.8.1" }, diff --git a/packages/instrumentation-together/package.json b/packages/instrumentation-together/package.json index c10b91fc..2d4301de 100644 --- a/packages/instrumentation-together/package.json +++ b/packages/instrumentation-together/package.json @@ -40,7 +40,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "js-tiktoken": "^1.0.20", "tslib": "^2.8.1" diff --git a/packages/instrumentation-vertexai/package.json b/packages/instrumentation-vertexai/package.json index 1c563a4a..6f0a72a2 100644 --- a/packages/instrumentation-vertexai/package.json +++ b/packages/instrumentation-vertexai/package.json @@ -40,7 +40,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.203.0", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "google-gax": "^4.0.0", "tslib": "^2.8.1" diff --git a/packages/traceloop-sdk/package.json b/packages/traceloop-sdk/package.json index 85568c10..2c77a5c6 100644 --- a/packages/traceloop-sdk/package.json +++ b/packages/traceloop-sdk/package.json @@ -63,7 +63,7 @@ "@opentelemetry/sdk-node": "^0.203.0", "@opentelemetry/sdk-trace-base": "^2.0.1", "@opentelemetry/sdk-trace-node": "^2.0.1", - "@opentelemetry/semantic-conventions": "^1.36.0", + "@opentelemetry/semantic-conventions": "^1.37.0", "@traceloop/ai-semantic-conventions": "workspace:*", "@traceloop/instrumentation-anthropic": "workspace:*", "@traceloop/instrumentation-bedrock": "workspace:*", diff --git a/packages/traceloop-sdk/src/lib/tracing/ai-sdk-transformations.ts b/packages/traceloop-sdk/src/lib/tracing/ai-sdk-transformations.ts index 7929a3fa..8263eca8 100644 --- a/packages/traceloop-sdk/src/lib/tracing/ai-sdk-transformations.ts +++ b/packages/traceloop-sdk/src/lib/tracing/ai-sdk-transformations.ts @@ -19,6 +19,10 @@ const AI_USAGE_PROMPT_TOKENS = "ai.usage.promptTokens"; const AI_USAGE_COMPLETION_TOKENS = "ai.usage.completionTokens"; const AI_MODEL_PROVIDER = "ai.model.provider"; const AI_PROMPT_TOOLS = "ai.prompt.tools"; +const TYPE_TEXT = "text"; +const TYPE_TOOL_CALL = "tool_call"; +const ROLE_ASSISTANT = "assistant"; +const ROLE_USER = "user"; // Vendor mapping from AI SDK provider prefixes to standardized LLM_SYSTEM values // Uses prefixes to match AI SDK patterns like "openai.chat", "anthropic.messages", etc. @@ -55,7 +59,21 @@ const transformResponseText = (attributes: Record): void => { if (AI_RESPONSE_TEXT in attributes) { attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.content`] = attributes[AI_RESPONSE_TEXT]; - attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = "assistant"; + attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = ROLE_ASSISTANT; + + const outputMessage = { + role: ROLE_ASSISTANT, + parts: [ + { + type: TYPE_TEXT, + content: attributes[AI_RESPONSE_TEXT], + }, + ], + }; + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] = JSON.stringify([ + outputMessage, + ]); + delete attributes[AI_RESPONSE_TEXT]; } }; @@ -64,7 +82,21 @@ const transformResponseObject = (attributes: Record): void => { if (AI_RESPONSE_OBJECT in attributes) { attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.content`] = attributes[AI_RESPONSE_OBJECT]; - attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = "assistant"; + attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = ROLE_ASSISTANT; + + const outputMessage = { + role: ROLE_ASSISTANT, + parts: [ + { + type: TYPE_TEXT, + content: attributes[AI_RESPONSE_OBJECT], + }, + ], + }; + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] = JSON.stringify([ + outputMessage, + ]); + delete attributes[AI_RESPONSE_OBJECT]; } }; @@ -76,8 +108,9 @@ const transformResponseToolCalls = (attributes: Record): void => { attributes[AI_RESPONSE_TOOL_CALLS] as string, ); - attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = "assistant"; + attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = ROLE_ASSISTANT; + const toolCallParts: any[] = []; toolCalls.forEach((toolCall: any, index: number) => { if (toolCall.toolCallType === "function") { attributes[ @@ -86,9 +119,27 @@ const transformResponseToolCalls = (attributes: Record): void => { attributes[ `${SpanAttributes.LLM_COMPLETIONS}.0.tool_calls.${index}.arguments` ] = toolCall.args; + + toolCallParts.push({ + type: TYPE_TOOL_CALL, + tool_call: { + name: toolCall.toolName, + arguments: toolCall.args, + }, + }); } }); + if (toolCallParts.length > 0) { + const outputMessage = { + role: ROLE_ASSISTANT, + parts: toolCallParts, + }; + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] = JSON.stringify([ + outputMessage, + ]); + } + delete attributes[AI_RESPONSE_TOOL_CALLS]; } catch { // Ignore parsing errors @@ -100,7 +151,10 @@ const processMessageContent = (content: any): string => { if (Array.isArray(content)) { const textItems = content.filter( (item: any) => - item && typeof item === "object" && item.type === "text" && item.text, + item && + typeof item === "object" && + item.type === TYPE_TEXT && + item.text, ); if (textItems.length > 0) { @@ -112,7 +166,7 @@ const processMessageContent = (content: any): string => { } if (content && typeof content === "object") { - if (content.type === "text" && content.text) { + if (content.type === TYPE_TEXT && content.text) { return content.text; } return JSON.stringify(content); @@ -126,7 +180,7 @@ const processMessageContent = (content: any): string => { (item: any) => item && typeof item === "object" && - item.type === "text" && + item.type === TYPE_TEXT && item.text, ); @@ -205,12 +259,32 @@ const transformPrompts = (attributes: Record): void => { } const messages = JSON.parse(jsonString); + const inputMessages: any[] = []; + messages.forEach((msg: { role: string; content: any }, index: number) => { const processedContent = processMessageContent(msg.content); const contentKey = `${SpanAttributes.LLM_PROMPTS}.${index}.content`; attributes[contentKey] = processedContent; attributes[`${SpanAttributes.LLM_PROMPTS}.${index}.role`] = msg.role; + + // Add to OpenTelemetry standard gen_ai.input.messages format + inputMessages.push({ + role: msg.role, + parts: [ + { + type: TYPE_TEXT, + content: processedContent, + }, + ], + }); }); + + // Set the OpenTelemetry standard input messages attribute + if (inputMessages.length > 0) { + attributes[SpanAttributes.LLM_INPUT_MESSAGES] = + JSON.stringify(inputMessages); + } + delete attributes[AI_PROMPT_MESSAGES]; } catch { // Ignore parsing errors @@ -223,7 +297,21 @@ const transformPrompts = (attributes: Record): void => { if (promptData.prompt && typeof promptData.prompt === "string") { attributes[`${SpanAttributes.LLM_PROMPTS}.0.content`] = promptData.prompt; - attributes[`${SpanAttributes.LLM_PROMPTS}.0.role`] = "user"; + attributes[`${SpanAttributes.LLM_PROMPTS}.0.role`] = ROLE_USER; + + const inputMessage = { + role: ROLE_USER, + parts: [ + { + type: TYPE_TEXT, + content: promptData.prompt, + }, + ], + }; + attributes[SpanAttributes.LLM_INPUT_MESSAGES] = JSON.stringify([ + inputMessage, + ]); + delete attributes[AI_PROMPT]; } } catch { diff --git a/packages/traceloop-sdk/test/ai-sdk-transformations.test.ts b/packages/traceloop-sdk/test/ai-sdk-transformations.test.ts index f50b1cbf..8ec61e68 100644 --- a/packages/traceloop-sdk/test/ai-sdk-transformations.test.ts +++ b/packages/traceloop-sdk/test/ai-sdk-transformations.test.ts @@ -1180,6 +1180,361 @@ describe("AI SDK Transformations", () => { }); }); + describe("transformAiSdkAttributes - gen_ai input/output messages", () => { + it("should create gen_ai.input.messages for conversation with text", () => { + const messages = [ + { role: "system", content: "You are a helpful assistant" }, + { role: "user", content: "Hello, how are you?" }, + { role: "assistant", content: "I'm doing well, thank you!" }, + { role: "user", content: "Can you help me with something?" }, + ]; + const attributes = { + "ai.prompt.messages": JSON.stringify(messages), + }; + + transformAiSdkAttributes(attributes); + + // Check that gen_ai.input.messages is properly set + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_INPUT_MESSAGES], + "string", + ); + + const inputMessages = JSON.parse( + attributes[SpanAttributes.LLM_INPUT_MESSAGES], + ); + assert.strictEqual(inputMessages.length, 4); + + // Check system message + assert.strictEqual(inputMessages[0].role, "system"); + assert.strictEqual(inputMessages[0].parts.length, 1); + assert.strictEqual(inputMessages[0].parts[0].type, "text"); + assert.strictEqual( + inputMessages[0].parts[0].content, + "You are a helpful assistant", + ); + + // Check user messages + assert.strictEqual(inputMessages[1].role, "user"); + assert.strictEqual( + inputMessages[1].parts[0].content, + "Hello, how are you?", + ); + + assert.strictEqual(inputMessages[2].role, "assistant"); + assert.strictEqual( + inputMessages[2].parts[0].content, + "I'm doing well, thank you!", + ); + + assert.strictEqual(inputMessages[3].role, "user"); + assert.strictEqual( + inputMessages[3].parts[0].content, + "Can you help me with something?", + ); + }); + + it("should create gen_ai.output.messages for text response", () => { + const attributes = { + "ai.response.text": "I'd be happy to help you with that!", + }; + + transformAiSdkAttributes(attributes); + + // Check that gen_ai.output.messages is properly set + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + "string", + ); + + const outputMessages = JSON.parse( + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + ); + assert.strictEqual(outputMessages.length, 1); + assert.strictEqual(outputMessages[0].role, "assistant"); + assert.strictEqual(outputMessages[0].parts.length, 1); + assert.strictEqual(outputMessages[0].parts[0].type, "text"); + assert.strictEqual( + outputMessages[0].parts[0].content, + "I'd be happy to help you with that!", + ); + }); + + it("should create gen_ai.output.messages for tool calls", () => { + const toolCallsData = [ + { + toolCallType: "function", + toolCallId: "call_weather_123", + toolName: "getWeather", + args: '{"location": "San Francisco", "unit": "celsius"}', + }, + { + toolCallType: "function", + toolCallId: "call_restaurant_456", + toolName: "findRestaurants", + args: '{"location": "San Francisco", "cuisine": "italian"}', + }, + ]; + + const attributes = { + "ai.response.toolCalls": JSON.stringify(toolCallsData), + }; + + transformAiSdkAttributes(attributes); + + // Check that gen_ai.output.messages is properly set + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + "string", + ); + + const outputMessages = JSON.parse( + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + ); + assert.strictEqual(outputMessages.length, 1); + assert.strictEqual(outputMessages[0].role, "assistant"); + assert.strictEqual(outputMessages[0].parts.length, 2); + + // Check first tool call + assert.strictEqual(outputMessages[0].parts[0].type, "tool_call"); + assert.strictEqual( + outputMessages[0].parts[0].tool_call.name, + "getWeather", + ); + assert.strictEqual( + outputMessages[0].parts[0].tool_call.arguments, + '{"location": "San Francisco", "unit": "celsius"}', + ); + + // Check second tool call + assert.strictEqual(outputMessages[0].parts[1].type, "tool_call"); + assert.strictEqual( + outputMessages[0].parts[1].tool_call.name, + "findRestaurants", + ); + assert.strictEqual( + outputMessages[0].parts[1].tool_call.arguments, + '{"location": "San Francisco", "cuisine": "italian"}', + ); + }); + + it("should create both gen_ai.input.messages and gen_ai.output.messages for complete conversation with tools", () => { + const inputMessages = [ + { + role: "system", + content: + "You are a helpful travel assistant. Use the available tools to help users plan their trips.", + }, + { + role: "user", + content: + "I'm planning a trip to San Francisco. Can you tell me about the weather and recommend some good Italian restaurants?", + }, + ]; + + const toolCallsData = [ + { + toolCallType: "function", + toolCallId: "call_weather_789", + toolName: "getWeather", + args: '{"location": "San Francisco", "forecast_days": 3}', + }, + { + toolCallType: "function", + toolCallId: "call_restaurants_101", + toolName: "searchRestaurants", + args: '{"location": "San Francisco", "cuisine": "italian", "rating_min": 4.0}', + }, + ]; + + const attributes = { + "ai.prompt.messages": JSON.stringify(inputMessages), + "ai.response.toolCalls": JSON.stringify(toolCallsData), + "ai.prompt.tools": [ + { + name: "getWeather", + description: "Get weather forecast for a location", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + forecast_days: { type: "number" }, + }, + required: ["location"], + }, + }, + { + name: "searchRestaurants", + description: "Search for restaurants in a location", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + cuisine: { type: "string" }, + rating_min: { type: "number" }, + }, + required: ["location"], + }, + }, + ], + }; + + transformAiSdkAttributes(attributes); + + // Check input messages + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_INPUT_MESSAGES], + "string", + ); + const parsedInputMessages = JSON.parse( + attributes[SpanAttributes.LLM_INPUT_MESSAGES], + ); + assert.strictEqual(parsedInputMessages.length, 2); + assert.strictEqual(parsedInputMessages[0].role, "system"); + assert.strictEqual( + parsedInputMessages[0].parts[0].content, + "You are a helpful travel assistant. Use the available tools to help users plan their trips.", + ); + assert.strictEqual(parsedInputMessages[1].role, "user"); + assert.strictEqual( + parsedInputMessages[1].parts[0].content, + "I'm planning a trip to San Francisco. Can you tell me about the weather and recommend some good Italian restaurants?", + ); + + // Check output messages (tool calls) + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + "string", + ); + const parsedOutputMessages = JSON.parse( + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + ); + assert.strictEqual(parsedOutputMessages.length, 1); + assert.strictEqual(parsedOutputMessages[0].role, "assistant"); + assert.strictEqual(parsedOutputMessages[0].parts.length, 2); + + // Verify tool calls in output + assert.strictEqual(parsedOutputMessages[0].parts[0].type, "tool_call"); + assert.strictEqual( + parsedOutputMessages[0].parts[0].tool_call.name, + "getWeather", + ); + assert.strictEqual(parsedOutputMessages[0].parts[1].type, "tool_call"); + assert.strictEqual( + parsedOutputMessages[0].parts[1].tool_call.name, + "searchRestaurants", + ); + + // Check that tools are also properly transformed + assert.strictEqual( + attributes[`${SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name`], + "getWeather", + ); + assert.strictEqual( + attributes[`${SpanAttributes.LLM_REQUEST_FUNCTIONS}.1.name`], + "searchRestaurants", + ); + }); + + it("should create gen_ai.output.messages for object response", () => { + const objectResponse = { + destination: "San Francisco", + weather: "sunny, 22°C", + recommendations: ["Visit Golden Gate Bridge", "Try local sourdough"], + confidence: 0.95, + }; + + const attributes = { + "ai.response.object": JSON.stringify(objectResponse), + }; + + transformAiSdkAttributes(attributes); + + // Check that gen_ai.output.messages is properly set + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + "string", + ); + + const outputMessages = JSON.parse( + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + ); + assert.strictEqual(outputMessages.length, 1); + assert.strictEqual(outputMessages[0].role, "assistant"); + assert.strictEqual(outputMessages[0].parts.length, 1); + assert.strictEqual(outputMessages[0].parts[0].type, "text"); + assert.strictEqual( + outputMessages[0].parts[0].content, + JSON.stringify(objectResponse), + ); + }); + + it("should handle complex multi-turn conversation with mixed content types", () => { + const complexMessages = [ + { + role: "system", + content: "You are an AI assistant that can analyze images and text.", + }, + { + role: "user", + content: [ + { type: "text", text: "What's in this image?" }, + { type: "image", url: "data:image/jpeg;base64,..." }, + ], + }, + { + role: "assistant", + content: "I can see a beautiful sunset over a mountain landscape.", + }, + { + role: "user", + content: + "Can you get the weather for this location using your tools?", + }, + ]; + + const attributes = { + "ai.prompt.messages": JSON.stringify(complexMessages), + }; + + transformAiSdkAttributes(attributes); + + // Check input messages transformation + const inputMessages = JSON.parse( + attributes[SpanAttributes.LLM_INPUT_MESSAGES], + ); + assert.strictEqual(inputMessages.length, 4); + + // System message should be preserved + assert.strictEqual(inputMessages[0].role, "system"); + assert.strictEqual( + inputMessages[0].parts[0].content, + "You are an AI assistant that can analyze images and text.", + ); + + // Complex content should be flattened to text parts only + assert.strictEqual(inputMessages[1].role, "user"); + assert.strictEqual( + inputMessages[1].parts[0].content, + "What's in this image?", + ); + + // Assistant response should be preserved + assert.strictEqual(inputMessages[2].role, "assistant"); + assert.strictEqual( + inputMessages[2].parts[0].content, + "I can see a beautiful sunset over a mountain landscape.", + ); + + // User follow-up should be preserved + assert.strictEqual(inputMessages[3].role, "user"); + assert.strictEqual( + inputMessages[3].parts[0].content, + "Can you get the weather for this location using your tools?", + ); + }); + }); + describe("transformAiSdkSpan", () => { it("should transform both span name and attributes", () => { const span = createMockSpan("ai.generateText.doGenerate", { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6dacfb9..4d9f2513 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,8 +106,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -155,8 +155,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -204,8 +204,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -250,8 +250,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -299,8 +299,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -363,8 +363,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -418,8 +418,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -482,8 +482,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -580,8 +580,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -632,8 +632,8 @@ importers: specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -783,8 +783,8 @@ importers: specifier: ^2.0.1 version: 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 + specifier: ^1.37.0 + version: 1.37.0 '@traceloop/ai-semantic-conventions': specifier: workspace:* version: link:../ai-semantic-conventions @@ -3311,8 +3311,8 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/semantic-conventions@1.36.0': - resolution: {integrity: sha512-TtxJSRD8Ohxp6bKkhrm27JRHAxPczQA7idtcTOMYI+wQRRrfgqxHv1cFbCApcSnNjtXkmzFozn6jQtFrOmbjPQ==} + '@opentelemetry/semantic-conventions@1.37.0': + resolution: {integrity: sha512-JD6DerIKdJGmRp4jQyX5FlrQjA4tjOw1cvfsPAZXfOOEErMUHjPcPSICS+6WnM0nB0efSFARh0KAZss+bvExOA==} engines: {node: '>=14'} '@phenomnomnominal/tsquery@5.0.1': @@ -11104,7 +11104,7 @@ snapshots: '@opentelemetry/core@2.0.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/semantic-conventions': 1.37.0 '@opentelemetry/exporter-logs-otlp-grpc@0.203.0(@opentelemetry/api@1.9.0)': dependencies: @@ -11209,7 +11209,7 @@ snapshots: '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.0.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/semantic-conventions': 1.37.0 '@opentelemetry/instrumentation@0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0)': dependencies: @@ -11259,7 +11259,7 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/semantic-conventions': 1.37.0 '@opentelemetry/sdk-logs@0.203.0(@opentelemetry/api@1.9.0)': dependencies: @@ -11298,7 +11298,7 @@ snapshots: '@opentelemetry/sdk-metrics': 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-node': 2.0.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/semantic-conventions': 1.37.0 transitivePeerDependencies: - supports-color @@ -11307,7 +11307,7 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 2.0.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/semantic-conventions': 1.37.0 '@opentelemetry/sdk-trace-node@2.0.1(@opentelemetry/api@1.9.0)': dependencies: @@ -11316,7 +11316,7 @@ snapshots: '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.0.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions@1.36.0': {} + '@opentelemetry/semantic-conventions@1.37.0': {} '@phenomnomnominal/tsquery@5.0.1(typescript@5.8.3)': dependencies: