[Inference] Add ASR support for Replicate provider (huggingface#1679)

Hello! This PR adds support for the `Automatic Speech Recognition` task
type for Replicate models.

Example:

-
[huggingface.co/openai/whisper-large-v3](https://huggingface.co/openai/whisper-large-v3)
- [replicate.com/openai/whisper](https://replicate.com/openai/whisper)

cc @hanouticelina

---------

Co-authored-by: Celina Hanouti <hanouticelina@gmail.com>
Co-authored-by: Eliott C. <coyotte508@gmail.com>

Author: 3 people

packages/inference/src/lib/getProviderHelper.ts

@@ -144,6 +144,7 @@ export const PROVIDERS: Record<InferenceProvider, Partial<Record<InferenceTask,
 		"text-to-speech": new Replicate.ReplicateTextToSpeechTask(),
 		"text-to-video": new Replicate.ReplicateTextToVideoTask(),
 		"image-to-image": new Replicate.ReplicateImageToImageTask(),
+		"automatic-speech-recognition": new Replicate.ReplicateAutomaticSpeechRecognitionTask(),
 	},
 	sambanova: {
 		conversational: new Sambanova.SambanovaConversationalTask(),

packages/inference/src/providers/replicate.ts

@@ -20,11 +20,14 @@ import type { BodyParams, HeaderParams, RequestArgs, UrlParams } from "../types.
 import { omit } from "../utils/omit.js";
 import {
 	TaskProviderHelper,
+	type AutomaticSpeechRecognitionTaskHelper,
 	type ImageToImageTaskHelper,
 	type TextToImageTaskHelper,
 	type TextToVideoTaskHelper,
 } from "./providerHelper.js";
 import type { ImageToImageArgs } from "../tasks/cv/imageToImage.js";
+import type { AutomaticSpeechRecognitionArgs } from "../tasks/audio/automaticSpeechRecognition.js";
+import type { AutomaticSpeechRecognitionOutput } from "@huggingface/tasks";
 import { base64FromBytes } from "../utils/base64FromBytes.js";
 export interface ReplicateOutput {
 	output?: string | string[];
@@ -163,6 +166,64 @@ export class ReplicateTextToVideoTask extends ReplicateTask implements TextToVid
 	}
 }
 
+export class ReplicateAutomaticSpeechRecognitionTask
+	extends ReplicateTask
+	implements AutomaticSpeechRecognitionTaskHelper
+{
+	override preparePayload(params: BodyParams): Record<string, unknown> {
+		return {
+			input: {
+				...omit(params.args, ["inputs", "parameters"]),
+				...(params.args.parameters as Record<string, unknown>),
+				audio: params.args.inputs, // This will be processed in preparePayloadAsync
+			},
+			version: params.model.includes(":") ? params.model.split(":")[1] : undefined,
+		};
+	}
+
+	async preparePayloadAsync(args: AutomaticSpeechRecognitionArgs): Promise<RequestArgs> {
+		const blob = "data" in args && args.data instanceof Blob ? args.data : "inputs" in args ? args.inputs : undefined;
+
+		if (!blob || !(blob instanceof Blob)) {
+			throw new Error("Audio input must be a Blob");
+		}
+
+		// Convert Blob to base64 data URL
+		const bytes = new Uint8Array(await blob.arrayBuffer());
+		const base64 = base64FromBytes(bytes);
+		const audioInput = `data:${blob.type || "audio/wav"};base64,${base64}`;
+
+		return {
+			...("data" in args ? omit(args, "data") : omit(args, "inputs")),
+			inputs: audioInput,
+		};
+	}
+
+	override async getResponse(response: ReplicateOutput): Promise<AutomaticSpeechRecognitionOutput> {
+		if (typeof response?.output === "string") return { text: response.output };
+		if (Array.isArray(response?.output) && typeof response.output[0] === "string") return { text: response.output[0] };
+
+		const out = response?.output as
+			| undefined
+			| {
+					transcription?: string;
+					translation?: string;
+					txt_file?: string;
+			  };
+		if (out && typeof out === "object") {
+			if (typeof out.transcription === "string") return { text: out.transcription };
+			if (typeof out.translation === "string") return { text: out.translation };
+			if (typeof out.txt_file === "string") {
+				const r = await fetch(out.txt_file);
+				return { text: await r.text() };
+			}
+		}
+		throw new InferenceClientProviderOutputError(
+			"Received malformed response from Replicate automatic-speech-recognition API"
+		);
+	}
+}
+
 export class ReplicateImageToImageTask extends ReplicateTask implements ImageToImageTaskHelper {
 	override preparePayload(params: BodyParams<ImageToImageArgs>): Record<string, unknown> {
 		return {

packages/inference/src/tasks/audio/automaticSpeechRecognition.ts

@@ -4,7 +4,6 @@ import { getProviderHelper } from "../../lib/getProviderHelper.js";
 import type { BaseArgs, Options } from "../../types.js";
 import { innerRequest } from "../../utils/request.js";
 import type { LegacyAudioInput } from "./utils.js";
-import { InferenceClientProviderOutputError } from "../../errors.js";
 
 export type AutomaticSpeechRecognitionArgs = BaseArgs & (AutomaticSpeechRecognitionInput | LegacyAudioInput);
 /**
@@ -22,9 +21,5 @@ export async function automaticSpeechRecognition(
 		...options,
 		task: "automatic-speech-recognition",
 	});
-	const isValidOutput = typeof res?.text === "string";
-	if (!isValidOutput) {
-		throw new InferenceClientProviderOutputError("Received malformed response from automatic-speech-recognition API");
-	}
 	return providerHelper.getResponse(res);
 }