Skip to content

Commit f7931b6

Browse files
minpeterminpeter
authored
FriendliAI: Documentation Updates (#7517)
* docs(friendliai.md): update FriendliAI documentation and model details * docs(friendliai.md): remove unused imports for cleaner documentation * feat: add support for parallel function calling, system messages, and response schema in model configuration
1 parent 2b0738e commit f7931b6

File tree

5 files changed

+47
-54
lines changed

5 files changed

+47
-54
lines changed

‎docs/my-website/docs/providers/friendliai.md

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,32 @@
11
# FriendliAI
2-
https://suite.friendli.ai/
32

3+
:::info
44
**We support ALL FriendliAI models, just set `friendliai/` as a prefix when sending completion requests**
5+
:::
6+
7+
| Property | Details |
8+
| -------------------------- | ----------------------------------------------------------------------------------------------- |
9+
| Description | The fastest and most efficient inference engine to build production-ready, compound AI systems. |
10+
| Provider Route on LiteLLM | `friendliai/` |
11+
| Provider Doc | [FriendliAI ↗](https://friendli.ai/docs/sdk/integrations/litellm) |
12+
| Supported OpenAI Endpoints | `/chat/completions`, `/completions` |
513

614
## API Key
15+
716
```python
817
# env variable
918
os.environ['FRIENDLI_TOKEN']
10-
os.environ['FRIENDLI_API_BASE'] # Optional. Set this when using dedicated endpoint.
1119
```
1220

1321
## Sample Usage
22+
1423
```python
1524
from litellm import completion
1625
import os
1726

1827
os.environ['FRIENDLI_TOKEN'] = ""
1928
response = completion(
20-
model="friendliai/mixtral-8x7b-instruct-v0-1",
29+
model="friendliai/meta-llama-3.1-8b-instruct",
2130
messages=[
2231
{"role": "user", "content": "hello from litellm"}
2332
],
@@ -26,13 +35,14 @@ print(response)
2635
```
2736

2837
## Sample Usage - Streaming
38+
2939
```python
3040
from litellm import completion
3141
import os
3242

3343
os.environ['FRIENDLI_TOKEN'] = ""
3444
response = completion(
35-
model="friendliai/mixtral-8x7b-instruct-v0-1",
45+
model="friendliai/meta-llama-3.1-8b-instruct",
3646
messages=[
3747
{"role": "user", "content": "hello from litellm"}
3848
],
@@ -43,18 +53,11 @@ for chunk in response:
4353
print(chunk)
4454
```
4555

46-
4756
## Supported Models
48-
### Serverless Endpoints
49-
We support ALL FriendliAI AI models, just set `friendliai/` as a prefix when sending completion requests
5057

51-
| Model Name | Function Call |
52-
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
53-
| mixtral-8x7b-instruct | `completion(model="friendliai/mixtral-8x7b-instruct-v0-1", messages)` |
54-
| meta-llama-3-8b-instruct | `completion(model="friendliai/meta-llama-3-8b-instruct", messages)` |
55-
| meta-llama-3-70b-instruct | `completion(model="friendliai/meta-llama-3-70b-instruct", messages)` |
58+
We support ALL FriendliAI AI models, just set `friendliai/` as a prefix when sending completion requests
5659

57-
### Dedicated Endpoints
58-
```
59-
model="friendliai/$ENDPOINT_ID:$ADAPTER_ROUTE"
60-
```
60+
| Model Name | Function Call |
61+
| --------------------------- | ---------------------------------------------------------------------- |
62+
| meta-llama-3.1-8b-instruct | `completion(model="friendliai/meta-llama-3.1-8b-instruct", messages)` |
63+
| meta-llama-3.1-70b-instruct | `completion(model="friendliai/meta-llama-3.1-70b-instruct", messages)` |

‎litellm/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -609,7 +609,7 @@ def add_known_models():
609609
"api.deepseek.com/v1",
610610
"api.together.xyz/v1",
611611
"app.empower.dev/api/v1",
612-
"inference.friendli.ai/v1",
612+
"https://api.friendli.ai/serverless/v1",
613613
"api.sambanova.ai/v1",
614614
"api.x.ai/v1",
615615
"api.galadriel.ai/v1",

‎litellm/litellm_core_utils/get_llm_provider_logic.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -208,11 +208,9 @@ def get_llm_provider( # noqa: PLR0915
208208
elif endpoint == "api.deepseek.com/v1":
209209
custom_llm_provider = "deepseek"
210210
dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
211-
elif endpoint == "inference.friendli.ai/v1":
211+
elif endpoint == "https://api.friendli.ai/serverless/v1":
212212
custom_llm_provider = "friendliai"
213-
dynamic_api_key = get_secret_str(
214-
"FRIENDLIAI_API_KEY"
215-
) or get_secret("FRIENDLI_TOKEN")
213+
dynamic_api_key = get_secret_str("FRIENDLIAI_API_KEY") or get_secret("FRIENDLI_TOKEN")
216214
elif endpoint == "api.galadriel.com/v1":
217215
custom_llm_provider = "galadriel"
218216
dynamic_api_key = get_secret_str("GALADRIEL_API_KEY")
@@ -550,7 +548,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
550548
api_base = (
551549
api_base
552550
or get_secret("FRIENDLI_API_BASE")
553-
or "https://inference.friendli.ai/v1"
551+
or "https://api.friendli.ai/serverless/v1"
554552
) # type: ignore
555553
dynamic_api_key = (
556554
api_key

‎litellm/model_prices_and_context_window_backup.json

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2264,35 +2264,31 @@
22642264
"mode": "chat",
22652265
"supports_function_calling": true
22662266
},
2267-
"friendliai/mixtral-8x7b-instruct-v0-1": {
2268-
"max_tokens": 32768,
2269-
"max_input_tokens": 32768,
2270-
"max_output_tokens": 32768,
2271-
"input_cost_per_token": 0.0000004,
2272-
"output_cost_per_token": 0.0000004,
2273-
"litellm_provider": "friendliai",
2274-
"mode": "chat",
2275-
"supports_function_calling": true
2276-
},
2277-
"friendliai/meta-llama-3-8b-instruct": {
2267+
"friendliai/meta-llama-3.1-8b-instruct": {
22782268
"max_tokens": 8192,
22792269
"max_input_tokens": 8192,
22802270
"max_output_tokens": 8192,
22812271
"input_cost_per_token": 0.0000001,
22822272
"output_cost_per_token": 0.0000001,
22832273
"litellm_provider": "friendliai",
22842274
"mode": "chat",
2285-
"supports_function_calling": true
2275+
"supports_function_calling": true,
2276+
"supports_parallel_function_calling": true,
2277+
"supports_system_messages": true,
2278+
"supports_response_schema": true
22862279
},
2287-
"friendliai/meta-llama-3-70b-instruct": {
2280+
"friendliai/meta-llama-3.1-70b-instruct": {
22882281
"max_tokens": 8192,
22892282
"max_input_tokens": 8192,
22902283
"max_output_tokens": 8192,
2291-
"input_cost_per_token": 0.0000008,
2292-
"output_cost_per_token": 0.0000008,
2284+
"input_cost_per_token": 0.0000006,
2285+
"output_cost_per_token": 0.0000006,
22932286
"litellm_provider": "friendliai",
22942287
"mode": "chat",
2295-
"supports_function_calling": true
2288+
"supports_function_calling": true,
2289+
"supports_parallel_function_calling": true,
2290+
"supports_system_messages": true,
2291+
"supports_response_schema": true
22962292
},
22972293
"claude-instant-1.2": {
22982294
"max_tokens": 8191,

‎model_prices_and_context_window.json

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2264,35 +2264,31 @@
22642264
"mode": "chat",
22652265
"supports_function_calling": true
22662266
},
2267-
"friendliai/mixtral-8x7b-instruct-v0-1": {
2268-
"max_tokens": 32768,
2269-
"max_input_tokens": 32768,
2270-
"max_output_tokens": 32768,
2271-
"input_cost_per_token": 0.0000004,
2272-
"output_cost_per_token": 0.0000004,
2273-
"litellm_provider": "friendliai",
2274-
"mode": "chat",
2275-
"supports_function_calling": true
2276-
},
2277-
"friendliai/meta-llama-3-8b-instruct": {
2267+
"friendliai/meta-llama-3.1-8b-instruct": {
22782268
"max_tokens": 8192,
22792269
"max_input_tokens": 8192,
22802270
"max_output_tokens": 8192,
22812271
"input_cost_per_token": 0.0000001,
22822272
"output_cost_per_token": 0.0000001,
22832273
"litellm_provider": "friendliai",
22842274
"mode": "chat",
2285-
"supports_function_calling": true
2275+
"supports_function_calling": true,
2276+
"supports_parallel_function_calling": true,
2277+
"supports_system_messages": true,
2278+
"supports_response_schema": true
22862279
},
2287-
"friendliai/meta-llama-3-70b-instruct": {
2280+
"friendliai/meta-llama-3.1-70b-instruct": {
22882281
"max_tokens": 8192,
22892282
"max_input_tokens": 8192,
22902283
"max_output_tokens": 8192,
2291-
"input_cost_per_token": 0.0000008,
2292-
"output_cost_per_token": 0.0000008,
2284+
"input_cost_per_token": 0.0000006,
2285+
"output_cost_per_token": 0.0000006,
22932286
"litellm_provider": "friendliai",
22942287
"mode": "chat",
2295-
"supports_function_calling": true
2288+
"supports_function_calling": true,
2289+
"supports_parallel_function_calling": true,
2290+
"supports_system_messages": true,
2291+
"supports_response_schema": true
22962292
},
22972293
"claude-instant-1.2": {
22982294
"max_tokens": 8191,

0 commit comments

Comments
 (0)