GPT-5 models handling + new verbosity parameter

Author: peterbanda

openai-client/src/main/scala/io/cequence/openaiscala/service/impl/EndPoint.scala

@@ -118,6 +118,7 @@ object Param {
   case object thread extends Param
   case object store extends Param
   case object reasoning_effort extends Param
+  case object verbosity extends Param
   case object service_tier extends Param
   case object web_search_options extends Param
   case object include extends Param

openai-client/src/main/scala/io/cequence/openaiscala/service/impl/OpenAIChatCompletionServiceImpl.scala

@@ -75,6 +75,16 @@ trait ChatCompletionBodyMaker {
     ModelId.o4_mini_2025_04_16
   )
 
+  private val gpt5Models = Set(
+    ModelId.gpt_5,
+    ModelId.gpt_5_2025_08_07,
+    ModelId.gpt_5_mini,
+    ModelId.gpt_5_mini_2025_08_07,
+    ModelId.gpt_5_nano,
+    ModelId.gpt_5_nano_2025_08_07,
+    ModelId.gpt_5_chat_latest
+  )
+
   protected def createBodyParamsForChatCompletion(
     messagesAux: Seq[BaseMessage],
     settings: CreateChatCompletionSettings,
@@ -91,12 +101,14 @@ trait ChatCompletionBodyMaker {
 
     val messageJsons = messagesFinal.map(Json.toJson(_)(messageWrites))
 
-    // regular O models need some special treatment... revisit this later
+    // revisit this later
     val settingsFinal =
       if (o1PreviewModels.contains(settings.model))
         ChatCompletionSettingsConversions.o1Preview(settings)
       else if (regularOModels.contains(settings.model))
         ChatCompletionSettingsConversions.o(settings)
+      else if (gpt5Models.contains(settings.model))
+        ChatCompletionSettingsConversions.gpt5(settings)
       else
         settings
 
@@ -136,6 +148,7 @@ trait ChatCompletionBodyMaker {
       Param.parallel_tool_calls -> settingsFinal.parallel_tool_calls,
       Param.store -> settingsFinal.store,
       Param.reasoning_effort -> settingsFinal.reasoning_effort.map(_.toString()),
+      Param.verbosity -> settingsFinal.verbosity.map(_.toString()),
       Param.service_tier -> settingsFinal.service_tier.map(_.toString()),
       Param.metadata -> (if (settingsFinal.metadata.nonEmpty) Some(settingsFinal.metadata)
                          else None),

openai-core/src/main/scala/io/cequence/openaiscala/JsonFormats.scala

@@ -36,6 +36,7 @@ import play.api.libs.json.JsonNaming.SnakeCase
 import play.api.libs.json.{Format, JsValue, Json, _}
 
 import java.{util => ju}
+import io.cequence.openaiscala.domain.settings.Verbosity
 
 object JsonFormats {
   private implicit lazy val dateFormat: Format[ju.Date] = JsonUtil.SecDateFormat
@@ -367,6 +368,12 @@ object JsonFormats {
     ReasoningEffort.high
   )
 
+  implicit val verbosityFormat: Format[Verbosity] = enumFormat[Verbosity](
+    Verbosity.low,
+    Verbosity.medium,
+    Verbosity.high
+  )
+
   implicit val serviceTierFormat: Format[ServiceTier] = enumFormat[ServiceTier](
     ServiceTier.auto,
     ServiceTier.default

openai-core/src/main/scala/io/cequence/openaiscala/OpenAIScalaClientException.scala

@@ -9,14 +9,18 @@ object Retryable {
   ): Option[OpenAIScalaClientException] = Some(t).filter(apply)
 
   def apply(t: OpenAIScalaClientException): Boolean = t match {
-    // we retry on these
-    case _: OpenAIScalaClientTimeoutException    => true
-    case _: OpenAIScalaRateLimitException        => true
-    case _: OpenAIScalaServerErrorException      => true
-    case _: OpenAIScalaEngineOverloadedException => true
+    // we don't retry on these
+    case _: OpenAIScalaClientUnknownHostException  => false
+    case _: OpenAIScalaTokenCountExceededException => false
+    case _: OpenAIScalaUnauthorizedException       => false
 
-    // otherwise don't retry
-    case _ => false
+    // we retry on these
+    case _: OpenAIScalaClientTimeoutException      => true
+    case _: OpenAIScalaRateLimitException          => true
+    case _: OpenAIScalaServerErrorException        => true
+    case _: OpenAIScalaEngineOverloadedException   => true
+    // generic case
+    case _: OpenAIScalaClientException             => true
   }
 }
 

openai-core/src/main/scala/io/cequence/openaiscala/domain/settings/CreateChatCompletionSettings.scala

@@ -86,6 +86,12 @@ case class CreateChatCompletionSettings(
   // Supported by o1 models only
   reasoning_effort: Option[ReasoningEffort] = None,
 
+  // Specifies the verbosity of the model's response. Only supported by gpt-5 models.
+  // Verbosity determines how many output tokens are generated. Lowering the number of tokens reduces overall latency. While the model's reasoning approach stays mostly the same, the model finds ways to answer more concisely—which can either improve or diminish answer quality, depending on your use case. Here are some scenarios for both ends of the verbosity spectrum:
+  // High verbosity: Use when you need the model to provide thorough explanations of documents or perform extensive code refactoring.
+  // Low verbosity: Best for situations where you want concise answers or simple code generation, such as SQL queries.
+  verbosity: Option[Verbosity] = None,
+
   // Specifies the latency tier to use for processing the request. This parameter is relevant for customers subscribed to the scale tier service:
   // If set to 'auto', and the Project is Scale tier enabled, the system will utilize scale tier credits until they are exhausted.
   // If set to 'auto', and the Project is not Scale tier enabled, the request will be processed using the default service tier with a lower uptime SLA and no latency guarentee.
@@ -141,6 +147,14 @@ object ReasoningEffort {
   case object high extends ReasoningEffort
 }
 
+sealed trait Verbosity extends EnumValue
+
+object Verbosity {
+  case object low extends Verbosity
+  case object medium extends Verbosity
+  case object high extends Verbosity
+}
+
 sealed trait ServiceTier extends EnumValue
 
 object ServiceTier {

openai-core/src/main/scala/io/cequence/openaiscala/service/OpenAIChatCompletionExtra.scala

@@ -22,6 +22,7 @@ import play.api.libs.json.{Format, JsValue, Json}
 
 import scala.concurrent.{ExecutionContext, Future}
 import com.fasterxml.jackson.core.JsonProcessingException
+import io.cequence.openaiscala.OpenAIScalaClientException
 
 object OpenAIChatCompletionExtra {
 
@@ -44,9 +45,28 @@ object OpenAIChatCompletionExtra {
     )(
       implicit ec: ExecutionContext,
       scheduler: Scheduler
+    ): Future[ChatCompletionResponse] = 
+      createChatCompletionWithFailoverSettings(
+        messages,
+        settings,
+        failoverModels.map(model => settings.copy(model = model)),
+        maxRetries,
+        retryOnAnyError,
+        failureMessage
+      )
+
+    def createChatCompletionWithFailoverSettings(
+      messages: Seq[BaseMessage],
+      settings: CreateChatCompletionSettings,
+      failoverSettings: Seq[CreateChatCompletionSettings],
+      maxRetries: Option[Int] = Some(defaultMaxRetries),
+      retryOnAnyError: Boolean = false,
+      failureMessage: String
+    )(
+      implicit ec: ExecutionContext,
+      scheduler: Scheduler
     ): Future[ChatCompletionResponse] = {
-      val failoverSettings = failoverModels.map(model => settings.copy(model = model))
-      val allSettingsInOrder = Seq(settings) ++ failoverSettings
+      val allSettingsInOrder = settings +: failoverSettings
 
       implicit val retrySettings: RetrySettings =
         RetrySettings(maxRetries = maxRetries.getOrElse(0))
@@ -152,7 +172,11 @@ object OpenAIChatCompletionExtra {
             s"${taskNameForLoggingFinal.capitalize} finished in " + (new java.util.Date().getTime - start.getTime) + " ms."
           )
 
-          json.as[T]
+          json.asOpt[T].getOrElse(
+            throw new OpenAIScalaClientException(
+              s"Failed to parse JSON response into the expected type. Response: $contentJson"
+            )
+          )
         }
     }
 
@@ -181,6 +205,13 @@ object OpenAIChatCompletionExtra {
   }
 
   private val defaultModelsSupportingJsonSchema = Seq(
+    ModelId.gpt_5,
+    ModelId.gpt_5_2025_08_07,
+    ModelId.gpt_5_mini,
+    ModelId.gpt_5_mini_2025_08_07,
+    ModelId.gpt_5_nano,
+    ModelId.gpt_5_nano_2025_08_07,
+    ModelId.gpt_5_chat_latest,
     ModelId.gpt_4_1,
     ModelId.gpt_4_1_2025_04_14,
     ModelId.gpt_4_1_mini,
@@ -194,6 +225,8 @@ object OpenAIChatCompletionExtra {
     ModelId.gpt_4o_2024_11_20,
     ModelId.o4_mini,
     ModelId.o4_mini_2025_04_16,
+    ModelId.o3_pro,
+    ModelId.o3_pro_2025_06_10,
     ModelId.o3,
     ModelId.o3_2025_04_16,
     ModelId.o3_mini,
@@ -203,9 +236,12 @@ object OpenAIChatCompletionExtra {
     ModelId.o1_2024_12_17,
     ModelId.o1_pro,
     ModelId.o1_pro_2025_03_19,
+    NonOpenAIModelId.gemini_2_5_pro,
+    NonOpenAIModelId.gemini_2_5_pro_preview_06_05,
     NonOpenAIModelId.gemini_2_5_pro_preview_05_06,
     NonOpenAIModelId.gemini_2_5_pro_preview_03_25,
     NonOpenAIModelId.gemini_2_5_pro_exp_03_25,
+    NonOpenAIModelId.gemini_2_5_flash,
     NonOpenAIModelId.gemini_2_5_flash_preview_05_20,
     NonOpenAIModelId.gemini_2_5_flash_preview_04_17,
     NonOpenAIModelId.gemini_2_5_flash_preview_04_17_thinking,

openai-core/src/main/scala/io/cequence/openaiscala/service/adapter/ChatCompletionSettingsConversions.scala

@@ -7,6 +7,7 @@ import io.cequence.openaiscala.domain.settings.{
 }
 import io.cequence.openaiscala.domain.settings.GroqCreateChatCompletionSettingsOps._
 import org.slf4j.LoggerFactory
+import io.cequence.openaiscala.domain.settings.Verbosity
 
 object ChatCompletionSettingsConversions {
 
@@ -17,24 +18,25 @@ object ChatCompletionSettingsConversions {
   case class FieldConversionDef(
     doConversion: CreateChatCompletionSettings => Boolean,
     convert: CreateChatCompletionSettings => CreateChatCompletionSettings,
-    loggingMessage: Option[String],
+    loggingMessage: Option[CreateChatCompletionSettings => String],
     warning: Boolean = false
   )
 
   def generic(
     fieldConversions: Seq[FieldConversionDef]
   ): SettingsConversion = (settings: CreateChatCompletionSettings) =>
     fieldConversions.foldLeft(settings) {
-      case (acc, FieldConversionDef(isDefined, convert, loggingMessage, warning)) =>
+      case (acc, FieldConversionDef(isDefined, convert, maybeLoggingMessage, warning)) =>
         if (isDefined(acc)) {
-          loggingMessage.foreach(message =>
+          maybeLoggingMessage.foreach { messageFun =>
+            val message = messageFun(acc)
             if (warning) logger.warn(message) else logger.debug(message)
-          )
+          }
           convert(acc)
         } else acc
     }
 
-  private val oBaseConversions = Seq(
+  private lazy val newAPIConversions = Seq(
     // max tokens
     FieldConversionDef(
       _.max_tokens.isDefined,
@@ -44,68 +46,85 @@ object ChatCompletionSettingsConversions {
           extra_params =
             settings.extra_params + ("max_completion_tokens" -> settings.max_tokens.get)
         ),
-      Some("O models don't support max_tokens, converting to max_completion_tokens")
+      Some(settings =>
+        s"${settings.model} model doesn't support max_tokens, converting to max_completion_tokens"
+      )
     ),
     // temperature
     FieldConversionDef(
       settings => settings.temperature.isDefined && settings.temperature.get != 1,
       _.copy(temperature = Some(1d)),
-      Some(
-        "O models don't support temperature values other than the default of 1, converting to 1."
+      Some(settings =>
+        s"${settings.model} model doesn't support temperature values other than the default of 1, converting to 1."
       ),
       warning = true
     ),
     // top_p
     FieldConversionDef(
       settings => settings.top_p.isDefined && settings.top_p.get != 1,
       _.copy(top_p = Some(1d)),
-      Some(
-        "O models don't support top p values other than the default of 1, converting to 1."
+      Some(settings =>
+        s"${settings.model} model doesn't support top p values other than the default of 1, converting to 1."
       ),
       warning = true
     ),
     // presence_penalty
     FieldConversionDef(
       settings => settings.presence_penalty.isDefined && settings.presence_penalty.get != 0,
       _.copy(presence_penalty = Some(0d)),
-      Some(
-        "O models don't support presence penalty values other than the default of 0, converting to 0."
+      Some(settings =>
+        s"${settings.model} model doesn't support presence penalty values other than the default of 0, converting to 0."
       ),
       warning = true
     ),
     // frequency_penalty
     FieldConversionDef(
       settings => settings.frequency_penalty.isDefined && settings.frequency_penalty.get != 0,
       _.copy(frequency_penalty = Some(0d)),
-      Some(
-        "O models don't support frequency penalty values other than the default of 0, converting to 0."
-      ),
-      warning = true
-    ),
-    // parallel_tool_calls
-    FieldConversionDef(
-      settings => settings.parallel_tool_calls.isDefined,
-      _.copy(parallel_tool_calls = None),
-      Some(
-        "O models don't support parallel tool calls, converting to None."
+      Some(settings =>
+        s"${settings.model} model doesn't support frequency penalty values other than the default of 0, converting to 0."
       ),
       warning = true
     )
   )
 
+  private lazy val oBaseConversions =
+    newAPIConversions ++ Seq(
+      // parallel_tool_calls
+      FieldConversionDef(
+        settings => settings.parallel_tool_calls.isDefined,
+        _.copy(parallel_tool_calls = None),
+        Some(settings =>
+          s"${settings.model} model doesn't support parallel tool calls, converting to None."
+        ),
+        warning = true
+      ),
+      // verbosity
+      FieldConversionDef(
+        settings => settings.verbosity.isDefined && settings.verbosity.get != Verbosity.medium,
+        _.copy(verbosity = None),
+        Some(settings =>
+          s"${settings.model} model doesn't support verbosity values other than 'medium', converting to None."
+        ),
+        warning = true
+      )
+    )
+
   private val o1PreviewConversions =
     oBaseConversions :+
       // response format type
       FieldConversionDef(
         settings =>
           settings.response_format_type.isDefined && settings.response_format_type.get != ChatCompletionResponseFormatType.text,
         _.copy(response_format_type = None),
-        Some(
-          "O1 (preview) models don't support json object/schema response format, converting to None."
+        Some(settings =>
+          s"O1 (preview) model ${settings.model} model doesn't support json object/schema response format, converting to None."
         ),
         warning = true
       )
 
+  val gpt5: SettingsConversion = generic(newAPIConversions)
+
   val o: SettingsConversion = generic(oBaseConversions)
 
   val o1Preview: SettingsConversion = generic(o1PreviewConversions)
@@ -125,8 +144,8 @@ object ChatCompletionSettingsConversions {
         ) && settings.max_tokens.isDefined,
       settings =>
         settings.copy(max_tokens = None).setMaxCompletionTokens(settings.max_tokens.get),
-      Some(
-        "Groq deepseek R1 model doesn't support max_tokens, converting to max_completion_tokens."
+      Some(settings =>
+        s"Groq deepseek R1 model ${settings.model} model doesn't support max_tokens, converting to max_completion_tokens."
       )
     ),
     // reasoning format
@@ -140,8 +159,8 @@ object ChatCompletionSettingsConversions {
           )
         ) && reasoningFormat.isDefined,
       _.setReasoningFormat(reasoningFormat.get),
-      Some(
-        s"Setting reasoning format '${reasoningFormat.get}' for Groq deepseek R1 model."
+      Some(settings =>
+        s"Setting reasoning format '${reasoningFormat.get}' for Groq deepseek R1 mode ${settings.model}."
       )
     )
   )