Skip to content

Commit ebad12d

Browse files
alexsin368alexsin368
authored
Merge branch 'main' into codetrans-remote-endpoint
2 parents 76e2dbe + f084b35 commit ebad12d

File tree

9 files changed

+81
-62
lines changed

9 files changed

+81
-62
lines changed

‎AgentQnA/docker_compose/intel/cpu/xeon/README.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ git clone https://github.com/opea-project/GenAIExamples.git
3333
cd GenAIExamples/AgentQnA
3434
```
3535

36-
Then checkout a released version, such as v1.4:
36+
To checkout a released version, such as v1.4:
3737

3838
```bash
3939
git checkout v1.4
@@ -49,7 +49,6 @@ export HF_TOKEN="Your_HuggingFace_API_Token" # the huggingface API token you ap
4949
export http_proxy="Your_HTTP_Proxy" # http proxy if any
5050
export https_proxy="Your_HTTPs_Proxy" # https proxy if any
5151
export no_proxy=localhost,127.0.0.1,$host_ip # additional no proxies if needed
52-
export NGINX_PORT=${your_nginx_port} # your usable port for nginx, 80 for example
5352
```
5453

5554
#### [Optional] OPENAI_API_KEY to use OpenAI models or LLM models with remote endpoints
@@ -67,7 +66,8 @@ export OPENAI_API_KEY=<your-openai-key>
6766
#### Then, set up environment variables for the selected hardware using the corresponding `set_env.sh`
6867

6968
```bash
70-
source $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon/set_env.sh
69+
cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon
70+
source ./set_env.sh
7171
```
7272

7373
### Deploy the Services Using Docker Compose
@@ -115,8 +115,8 @@ docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/
115115
Please refer to the table below to build different microservices from source:
116116

117117
| Microservice | Deployment Guide |
118-
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | --- |
119-
| Agent | [Agent build guide](https://github.com/opea-project/GenAIComps/blob/main/comps/agent/src/README.md#21-build-docker-image-for-agent-microservice) | |
118+
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ |
119+
| Agent | [Agent build guide](https://github.com/opea-project/GenAIComps/blob/main/comps/agent/src/README.md#21-build-docker-image-for-agent-microservice) |
120120
| UI | [Basic UI build guide](../../../../README_miscellaneous.md#build-ui-docker-image) |
121121

122122
### Ingest Data into the Vector Database

‎AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,11 +101,18 @@ services:
101101
- "8080:8000"
102102
ipc: host
103103
agent-ui:
104-
image: opea/agent-ui
104+
image: ghcr.io/open-webui/open-webui:main
105105
container_name: agent-ui
106106
ports:
107107
- "5173:8080"
108108
ipc: host
109+
environment:
110+
no_proxy: ${no_proxy}
111+
http_proxy: ${http_proxy}
112+
https_proxy: ${https_proxy}
113+
OPENAI_API_KEYS: "empty"
114+
OPENAI_API_BASE_URLS: ${SUPERVISOR_AGENT_ENDPOINT}
115+
ENABLE_OLLAMA_API: False
109116

110117
networks:
111118
default:

‎AgentQnA/docker_compose/intel/cpu/xeon/compose_remote.yaml

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,49 @@
22
# SPDX-License-Identifier: Apache-2.0
33

44
services:
5+
agent-vllm-service:
6+
image: ${REGISTRY:-opea}/vllm:${TAG:-latest}
7+
container_name: vllm-service
8+
ports:
9+
- "9009:80"
10+
volumes:
11+
- "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub"
12+
shm_size: 128g
13+
environment:
14+
no_proxy: ${no_proxy}
15+
http_proxy: ${http_proxy}
16+
https_proxy: ${https_proxy}
17+
HF_TOKEN: ${HF_TOKEN}
18+
LLM_MODEL_ID: ${model}
19+
VLLM_TORCH_PROFILER_DIR: "/mnt"
20+
VLLM_CPU_KVCACHE_SPACE: 40
21+
healthcheck:
22+
test: ["CMD-SHELL", "curl -f http://localhost:80/health || exit 1"]
23+
interval: 10s
24+
timeout: 10s
25+
retries: 100
26+
command: --model $model --host 0.0.0.0 --port 80
27+
528
worker-rag-agent:
29+
depends_on:
30+
agent-vllm-service:
31+
condition: service_healthy
632
environment:
7-
llm_endpoint_url: ${REMOTE_ENDPOINT}
8-
api_key: ${OPENAI_API_KEY}
33+
llm_engine: vllm
34+
llm_endpoint_url: ${LLM_ENDPOINT_URL}
935

1036
worker-sql-agent:
37+
depends_on:
38+
agent-vllm-service:
39+
condition: service_healthy
1140
environment:
12-
llm_endpoint_url: ${REMOTE_ENDPOINT}
13-
api_key: ${OPENAI_API_KEY}
41+
llm_engine: vllm
42+
llm_endpoint_url: ${LLM_ENDPOINT_URL}
1443

1544
supervisor-react-agent:
45+
depends_on:
46+
agent-vllm-service:
47+
condition: service_healthy
1648
environment:
17-
llm_endpoint_url: ${REMOTE_ENDPOINT}
18-
api_key: ${OPENAI_API_KEY}
49+
llm_engine: vllm
50+
llm_endpoint_url: ${LLM_ENDPOINT_URL}

‎AgentQnA/docker_compose/intel/cpu/xeon/set_env.sh

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,11 @@ export OPENAI_API_KEY=${OPENAI_API_KEY}
2828
export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions"
2929
export SQL_AGENT_URL="http://${ip_address}:9096/v1/chat/completions"
3030
export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
31-
export CRAG_SERVER=http://${ip_address}:8080
31+
export LLM_ENDPOINT_URL="http://${ip_address}:9009"
32+
export CRAG_SERVER="http://${ip_address}:8080"
3233
export db_name=Chinook
3334
export db_path="sqlite:////home/user/chinook-db/Chinook_Sqlite.sqlite"
35+
export SUPERVISOR_AGENT_ENDPOINT="http://${ip_address}:9090/v1"
3436

3537
if [ ! -f $WORKDIR/GenAIExamples/AgentQnA/tests/Chinook_Sqlite.sqlite ]; then
3638
echo "Download Chinook_Sqlite!"
@@ -61,4 +63,4 @@ export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get"
6163
export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete"
6264

6365

64-
export no_proxy="$no_proxy,rag-agent-endpoint,sql-agent-endpoint,react-agent-endpoint,agent-ui"
66+
export no_proxy="$no_proxy,$host_ip,rag-agent-endpoint,sql-agent-endpoint,react-agent-endpoint,agent-ui,vllm-service"

‎AgentQnA/docker_compose/intel/hpu/gaudi/README.md

Lines changed: 3 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ git clone https://github.com/opea-project/GenAIExamples.git
3333
cd GenAIExamples/AgentQnA
3434
```
3535

36-
Then checkout a released version, such as v1.4:
36+
To checkout a released version, such as v1.4:
3737

3838
```bash
3939
git checkout v1.4
@@ -49,7 +49,6 @@ export HF_TOKEN="Your_HuggingFace_API_Token" # the huggingface API token you ap
4949
export http_proxy="Your_HTTP_Proxy" # http proxy if any
5050
export https_proxy="Your_HTTPs_Proxy" # https proxy if any
5151
export no_proxy=localhost,127.0.0.1,$host_ip # additional no proxies if needed
52-
export NGINX_PORT=${your_nginx_port} # your usable port for nginx, 80 for example
5352
```
5453

5554
#### [Optional] OPENAI_API_KEY to use OpenAI models or Intel® AI for Enterprise Inference
@@ -107,37 +106,13 @@ docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/
107106

108107
</details>
109108

110-
#### OpenAI Models
111-
112-
The command below will launch the multi-agent system with the `DocIndexRetriever` as the retrieval tool for the Worker RAG agent.
113-
114-
```bash
115-
docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose_openai.yaml up -d
116-
```
117-
118-
#### Models on Remote Server
119-
120-
When models are deployed on a remote server with Intel® AI for Enterprise Inference, a base URL and an API key are required to access them. To run the Agent microservice on Gaudi while using models deployed on a remote server, add `compose_remote.yaml` to the `docker compose` command and set additional environment variables.
121-
122-
#### Notes
123-
124-
- `OPENAI_API_KEY` is already set in a previous step.
125-
- `model` is used to overwrite the value set for this environment variable in `set_env.sh`.
126-
- `LLM_ENDPOINT_URL` is the base URL given from the owner of the on-prem machine or cloud service provider. It will follow this format: "https://<DNS>". Here is an example: "https://api.inference.example.com".
127-
128-
```bash
129-
export model=<name-of-model-card>
130-
export LLM_ENDPOINT_URL=<http-endpoint-of-remote-server>
131-
docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose_openai.yaml -f compose_remote.yaml up -d
132-
```
133-
134109
#### Build image from source
135110

136111
Please refer to the table below to build different microservices from source:
137112

138113
| Microservice | Deployment Guide |
139-
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | --- |
140-
| Agent | [Agent build guide](https://github.com/opea-project/GenAIComps/blob/main/comps/agent/src/README.md#21-build-docker-image-for-agent-microservice) | |
114+
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ |
115+
| Agent | [Agent build guide](https://github.com/opea-project/GenAIComps/blob/main/comps/agent/src/README.md#21-build-docker-image-for-agent-microservice) |
141116
| UI | [Basic UI build guide](../../../../README_miscellaneous.md#build-ui-docker-image) |
142117

143118
### Ingest Data into the Vector Database

‎AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ services:
1010
ports:
1111
- "9095:9095"
1212
ipc: host
13+
depends_on:
14+
vllm-service:
15+
condition: service_healthy
1316
environment:
1417
ip_address: ${ip_address}
1518
strategy: rag_agent_llama
@@ -41,6 +44,9 @@ services:
4144
ports:
4245
- "9096:9096"
4346
ipc: host
47+
depends_on:
48+
vllm-service:
49+
condition: service_healthy
4450
environment:
4551
ip_address: ${ip_address}
4652
strategy: sql_agent_llama
@@ -66,8 +72,12 @@ services:
6672
image: ${REGISTRY:-opea}/agent:${TAG:-latest}
6773
container_name: react-agent-endpoint
6874
depends_on:
69-
- worker-rag-agent
70-
- worker-sql-agent
75+
worker-rag-agent:
76+
condition: service_started
77+
worker-sql-agent:
78+
condition: service_started
79+
vllm-service:
80+
condition: service_healthy
7181
volumes:
7282
- ${TOOLSET_PATH:-../../../../tools}:/home/user/tools/
7383
ports:
@@ -106,11 +116,17 @@ services:
106116
agent-ui:
107117
image: ${REGISTRY:-opea}/agent-ui:${TAG:-latest}
108118
container_name: agent-ui
109-
environment:
110-
host_ip: ${host_ip}
111119
ports:
112120
- "5173:8080"
113121
ipc: host
122+
environment:
123+
host_ip: ${host_ip}
124+
no_proxy: ${no_proxy}
125+
http_proxy: ${http_proxy}
126+
https_proxy: ${https_proxy}
127+
OPENAI_API_KEYS: "empty"
128+
OPENAI_API_BASE_URLS: ${SUPERVISOR_AGENT_ENDPOINT}
129+
ENABLE_OLLAMA_API: False
114130
vllm-service:
115131
image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest}
116132
container_name: vllm-gaudi-server

‎AgentQnA/docker_compose/intel/hpu/gaudi/set_env.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions"
3636
export SQL_AGENT_URL="http://${ip_address}:9096/v1/chat/completions"
3737
export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
3838
export CRAG_SERVER=http://${ip_address}:8080
39+
export SUPERVISOR_AGENT_ENDPOINT="http://${ip_address}:9090/v1"
3940

4041
export db_name=Chinook
4142
export db_path="sqlite:////home/user/chinook-db/Chinook_Sqlite.sqlite"

‎CodeGen/ui/docker/Dockerfile.gradio

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,9 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missin
1212
default-jre \
1313
libgl1-mesa-glx \
1414
libjemalloc-dev \
15+
ffmpeg \
1516
wget
1617

17-
# Install ffmpeg static build
18-
WORKDIR /root
19-
RUN wget https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-amd64-static.tar.xz && \
20-
mkdir ffmpeg-git-amd64-static && tar -xvf ffmpeg-git-amd64-static.tar.xz -C ffmpeg-git-amd64-static --strip-components 1 && \
21-
export PATH=/root/ffmpeg-git-amd64-static:$PATH && \
22-
cp /root/ffmpeg-git-amd64-static/ffmpeg /usr/local/bin/ && \
23-
cp /root/ffmpeg-git-amd64-static/ffprobe /usr/local/bin/
24-
2518
RUN mkdir -p /home/user
2619

2720
COPY gradio /home/user/gradio

‎MultimodalQnA/ui/docker/Dockerfile

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,9 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missin
1212
default-jre \
1313
libgl1-mesa-glx \
1414
libjemalloc-dev \
15+
ffmpeg \
1516
wget
1617

17-
# Install ffmpeg static build
18-
WORKDIR /root
19-
RUN wget https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-amd64-static.tar.xz && \
20-
mkdir ffmpeg-git-amd64-static && tar -xvf ffmpeg-git-amd64-static.tar.xz -C ffmpeg-git-amd64-static --strip-components 1 && \
21-
export PATH=/root/ffmpeg-git-amd64-static:$PATH && \
22-
cp /root/ffmpeg-git-amd64-static/ffmpeg /usr/local/bin/ && \
23-
cp /root/ffmpeg-git-amd64-static/ffprobe /usr/local/bin/
24-
2518
RUN mkdir -p /home/user
2619

2720
COPY gradio /home/user/gradio

0 commit comments

Comments
 (0)