Skip to content

Commit db2d2bd

Browse files
chensuyuechensuyue
authoredAug 20, 2024··
fix chatqna guardrails (#615)
Signed-off-by: chensuyue <[email protected]> Signed-off-by: letonghan <[email protected]>
·
v1.2v0.9
1 parent 4fa37e7 commit db2d2bd

File tree

3 files changed

+71
-69
lines changed

3 files changed

+71
-69
lines changed
 

‎ChatQnA/docker/chatqna_guardrails.py

+11-10
Original file line numberDiff line numberDiff line change
@@ -66,20 +66,21 @@ def add_remote_service(self):
6666
use_remote_service=True,
6767
service_type=ServiceType.LLM,
6868
)
69-
guardrail_out = MicroService(
70-
name="guardrail_out",
71-
host=GUARDRAIL_SERVICE_HOST_IP,
72-
port=GUARDRAIL_SERVICE_PORT,
73-
endpoint="/v1/guardrails",
74-
use_remote_service=True,
75-
service_type=ServiceType.GUARDRAIL,
76-
)
77-
self.megaservice.add(guardrail_in).add(embedding).add(retriever).add(rerank).add(llm).add(guardrail_out)
69+
# guardrail_out = MicroService(
70+
# name="guardrail_out",
71+
# host=GUARDRAIL_SERVICE_HOST_IP,
72+
# port=GUARDRAIL_SERVICE_PORT,
73+
# endpoint="/v1/guardrails",
74+
# use_remote_service=True,
75+
# service_type=ServiceType.GUARDRAIL,
76+
# )
77+
# self.megaservice.add(guardrail_in).add(embedding).add(retriever).add(rerank).add(llm).add(guardrail_out)
78+
self.megaservice.add(guardrail_in).add(embedding).add(retriever).add(rerank).add(llm)
7879
self.megaservice.flow_to(guardrail_in, embedding)
7980
self.megaservice.flow_to(embedding, retriever)
8081
self.megaservice.flow_to(retriever, rerank)
8182
self.megaservice.flow_to(rerank, llm)
82-
self.megaservice.flow_to(llm, guardrail_out)
83+
# self.megaservice.flow_to(llm, guardrail_out)
8384
self.gateway = ChatQnAGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
8485

8586

‎ChatQnA/docker/docker_build_compose.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ services:
1010
no_proxy: ${no_proxy}
1111
dockerfile: ./Dockerfile
1212
image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
13+
chatqna-guardrails:
14+
build:
15+
dockerfile: ./Dockerfile_guardrails
16+
extends: chatqna
17+
image: ${REGISTRY:-opea}/chatqna-guardrails:${TAG:-latest}
1318
chatqna-ui:
1419
build:
1520
context: ui
@@ -94,6 +99,12 @@ services:
9499
dockerfile: comps/dataprep/qdrant/docker/Dockerfile
95100
extends: chatqna
96101
image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}
102+
guardrails-tgi:
103+
build:
104+
context: GenAIComps
105+
dockerfile: comps/guardrails/llama_guard/docker/Dockerfile
106+
extends: chatqna
107+
image: ${REGISTRY:-opea}/guardrails-tgi:${TAG:-latest}
97108
tei-gaudi:
98109
build:
99110
context: tei-gaudi

‎ChatQnA/tests/_test_chatqna_guardrails_on_gaudi.sh ‎ChatQnA/tests/test_chatqna_guardrails_on_gaudi.sh

+49-59
Original file line numberDiff line numberDiff line change
@@ -3,48 +3,33 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -e
6-
echo "IMAGE_REPO=${IMAGE_REPO}"
6+
IMAGE_REPO=${IMAGE_REPO:-"opea"}
7+
IMAGE_TAG=${IMAGE_TAG:-"latest"}
8+
echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
9+
echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
10+
export REGISTRY=${IMAGE_REPO}
11+
export TAG=${IMAGE_TAG}
712

813
WORKPATH=$(dirname "$PWD")
914
LOG_PATH="$WORKPATH/tests"
1015
ip_address=$(hostname -I | awk '{print $1}')
1116

1217
function build_docker_images() {
13-
cd $WORKPATH
18+
cd $WORKPATH/docker
1419
git clone https://github.com/opea-project/GenAIComps.git
15-
cd GenAIComps
16-
17-
docker build -t opea/embedding-tei:latest -f comps/embeddings/langchain/docker/Dockerfile .
18-
docker build -t opea/retriever-redis:latest -f comps/retrievers/langchain/redis/docker/Dockerfile .
19-
docker build -t opea/reranking-tei:latest -f comps/reranks/tei/docker/Dockerfile .
20-
docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
21-
docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/docker/Dockerfile .
22-
docker build -t opea/guardrails-tgi:latest -f comps/guardrails/llama_guard/docker/Dockerfile .
20+
git clone https://github.com/huggingface/tei-gaudi
2321

24-
# cd ..
25-
# git clone https://github.com/huggingface/tei-gaudi
26-
# cd tei-gaudi/
27-
# docker build --no-cache -f Dockerfile-hpu -t opea/tei-gaudi:latest .
22+
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
23+
service_list="chatqna-guardrails chatqna-ui dataprep-redis embedding-tei retriever-redis reranking-tei llm-tgi tei-gaudi guardrails-tgi"
24+
docker compose -f docker_build_compose.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2825

2926
docker pull ghcr.io/huggingface/tgi-gaudi:2.0.1
3027
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
3128

32-
cd $WORKPATH/docker
33-
docker build --no-cache -t opea/chatqna-guardrails:latest -f Dockerfile_guardrails .
34-
35-
cd $WORKPATH/docker/ui
36-
docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile .
37-
3829
docker images
3930
}
4031

4132
function start_services() {
42-
# build tei-gaudi for each test instead of pull from local registry
43-
cd $WORKPATH
44-
git clone https://github.com/huggingface/tei-gaudi
45-
cd tei-gaudi/
46-
docker build --no-cache -q -f Dockerfile-hpu -t opea/tei-gaudi:latest .
47-
4833
cd $WORKPATH/docker/gaudi
4934
export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
5035
export RERANK_MODEL_ID="BAAI/bge-reranker-base"
@@ -69,18 +54,6 @@ function start_services() {
6954

7055
sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
7156

72-
if [[ "$IMAGE_REPO" != "" ]]; then
73-
# Replace the container name with a test-specific name
74-
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
75-
sed -i "s#image: opea/chatqna-guardrails:latest#image: opea/chatqna:${IMAGE_TAG}#g" compose_guardrails.yaml
76-
sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" compose_guardrails.yaml
77-
sed -i "s#image: opea/chatqna-conversation-ui:latest#image: opea/chatqna-conversation-ui:${IMAGE_TAG}#g" compose_guardrails.yaml
78-
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" compose_guardrails.yaml
79-
sed -i "s#image: ${IMAGE_REPO}opea/tei-gaudi:latest#image: opea/tei-gaudi:latest#g" compose_guardrails.yaml
80-
echo "cat compose_guardrails.yaml"
81-
cat compose_guardrails.yaml
82-
fi
83-
8457
# Start Docker Containers
8558
docker compose -f compose_guardrails.yaml up -d
8659
n=0
@@ -92,6 +65,17 @@ function start_services() {
9265
sleep 1s
9366
n=$((n+1))
9467
done
68+
69+
# Make sure tgi guardrails service is ready
70+
n=0
71+
until [[ "$n" -ge 400 ]]; do
72+
docker logs tgi-guardrails-server > tgi_guardrails_service_start.log
73+
if grep -q Connected tgi_guardrails_service_start.log; then
74+
break
75+
fi
76+
sleep 1s
77+
n=$((n+1))
78+
done
9579
}
9680

9781
function validate_services() {
@@ -101,24 +85,27 @@ function validate_services() {
10185
local DOCKER_NAME="$4"
10286
local INPUT_DATA="$5"
10387

104-
local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
105-
if [ "$HTTP_STATUS" -eq 200 ]; then
106-
echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
88+
HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
89+
HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
90+
RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
10791

108-
local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
92+
docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
10993

110-
if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
111-
echo "[ $SERVICE_NAME ] Content is as expected."
112-
else
113-
echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
114-
docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
115-
exit 1
116-
fi
117-
else
94+
# check response status
95+
if [ "$HTTP_STATUS" -ne "200" ]; then
11896
echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
119-
docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
12097
exit 1
98+
else
99+
echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
121100
fi
101+
# check response body
102+
if [[ "$RESPONSE_BODY" != *"$EXPECTED_RESULT"* ]]; then
103+
echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY"
104+
exit 1
105+
else
106+
echo "[ $SERVICE_NAME ] Content is as expected."
107+
fi
108+
122109
sleep 1s
123110
}
124111

@@ -128,15 +115,15 @@ function validate_microservices() {
128115
# tei for embedding service
129116
validate_services \
130117
"${ip_address}:8090/embed" \
131-
"\[\[" \
118+
"[[" \
132119
"tei-embedding" \
133120
"tei-embedding-gaudi-server" \
134121
'{"inputs":"What is Deep Learning?"}'
135122

136123
# embedding microservice
137124
validate_services \
138125
"${ip_address}:6000/v1/embeddings" \
139-
'"text":"What is Deep Learning?","embedding":\[' \
126+
'"text":"What is Deep Learning?","embedding":[' \
140127
"embedding" \
141128
"embedding-tei-server" \
142129
'{"text":"What is Deep Learning?"}'
@@ -147,7 +134,7 @@ function validate_microservices() {
147134
test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
148135
validate_services \
149136
"${ip_address}:7000/v1/retrieval" \
150-
" " \
137+
"retrieved_docs" \
151138
"retrieval" \
152139
"retriever-redis-server" \
153140
"{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}"
@@ -186,7 +173,7 @@ function validate_microservices() {
186173

187174
# tgi for guardrails service
188175
validate_services \
189-
"${ip_address}:8008/generate" \
176+
"${ip_address}:8088/generate" \
190177
"generated_text" \
191178
"tgi-guardrails" \
192179
"tgi-guardrails-server" \
@@ -217,13 +204,16 @@ function validate_frontend() {
217204
cd $WORKPATH/docker/ui/svelte
218205
local conda_env_name="OPEA_e2e"
219206
export PATH=${HOME}/miniforge3/bin/:$PATH
220-
# conda remove -n ${conda_env_name} --all -y
221-
# conda create -n ${conda_env_name} python=3.12 -y
207+
if conda info --envs | grep -q "$conda_env_name"; then
208+
echo "$conda_env_name exist!"
209+
else
210+
conda create -n ${conda_env_name} python=3.12 -y
211+
fi
222212
source activate ${conda_env_name}
223213

224214
sed -i "s/localhost/$ip_address/g" playwright.config.ts
225215

226-
# conda install -c conda-forge nodejs -y
216+
conda install -c conda-forge nodejs -y
227217
npm install && npm ci && npx playwright install --with-deps
228218
node -v && npm -v && pip list
229219

@@ -246,7 +236,7 @@ function stop_docker() {
246236
function main() {
247237

248238
stop_docker
249-
if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
239+
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
250240
start_time=$(date +%s)
251241
start_services
252242
end_time=$(date +%s)

0 commit comments

Comments
 (0)
Please sign in to comment.