Skip to content

Commit a06ebd0

Browse files
changdazhouchangdazhou
committed
update paddlex ci
1 parent 7738bd0 commit a06ebd0

File tree

4 files changed

+89
-62
lines changed

4 files changed

+89
-62
lines changed

‎models/PaddleX/ci/black_list.txt

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,11 @@
11
################################### All ##################################
2-
All:PP-YOLOE_plus_SOD-largesize-L
32
################################## Train #################################
43
################################ Evaluate ################################
54
################################# Predict ################################
6-
Predict:PP-YOLOE_seg-S
7-
Predict:MaskRCNN-ResNet50
8-
Predict:MaskRCNN-ResNet50-FPN
9-
Predict:MaskRCNN-ResNet50-vd-FPN
10-
Predict:MaskRCNN-ResNet101-FPN
11-
Predict:MaskRCNN-ResNet101-vd-FPN
12-
Predict:MaskRCNN-ResNeXt101-vd-FPN
13-
Predict:Cascade-MaskRCNN-ResNet50-vd-SSLDv2-FPN
14-
Predict:Cascade-MaskRCNN-ResNet50-FPN
15-
Predict:Nonstationary_ad
165
Predict:TimesNet_ad
17-
Predict:PatchTST_ad
18-
Predict:DLinear_ad
19-
Predict:PP-YOLOE_plus_SOD-L
20-
Predict:PP-YOLOE_plus_SOD-S
21-
Predict:Cascade-FasterRCNN-ResNet50-vd-SSLDv2-FPN
22-
Predict:Cascade-FasterRCNN-ResNet50-FPN
23-
Predict:FasterRCNN-Swin-Tiny-FPN
24-
Predict:FasterRCNN-ResNeXt101-vd-FPN
25-
Predict:FasterRCNN-ResNet101-FPN
26-
Predict:FasterRCNN-ResNet101
27-
Predict:FasterRCNN-ResNet50-vd-SSLDv2-FPN
28-
Predict:FasterRCNN-ResNet50-vd-FPN
29-
Predict:FasterRCNN-ResNet50-FPN
30-
Predict:FasterRCNN-ResNet50
31-
Predict:FasterRCNN-ResNet34-FPN
326
################################## Export ################################
337
################################# Pipeline ###############################
348
Pipeline:PP-ChatOCRv3-doc
359
Pipeline:PP-ShiTuV2
3610
Pipeline:face_recognition
11+
Pipeline:ts_ad

‎models/PaddleX/ci/checker.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -227,8 +227,8 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso
227227
pass_flag = False
228228
else:
229229
show_type = json_data.get("show_type")
230-
if show_type not in ["image", "txt", "csv"]:
231-
check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv'其中一个")
230+
if show_type not in ["image", "txt", "csv", "video"]:
231+
check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv','video'其中一个")
232232
pass_flag = False
233233

234234
if module_name == "general_recognition":
@@ -249,6 +249,7 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso
249249
or "table" in module_name
250250
or "formula" in module_name
251251
or module_name == "general_recognition"
252+
or module_name == "face_feature"
252253
):
253254
try:
254255
num_class = int(json_data["attributes"].get("num_classes"))
@@ -368,7 +369,7 @@ def run_checks(self, args):
368369

369370
if args.check_train_result_json:
370371
# 检查 train_result.json 内容
371-
train_result_json = os.path.join(output_dir, "train_results.json")
372+
train_result_json = os.path.join(output_dir, "train_result.json")
372373
check_weights_items = args.check_weights_items
373374
check_train_json_message = []
374375
check_train_json_flag, check_train_json_message = self.check_train_json_content(

‎models/PaddleX/ci/ci_run.sh

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ MODEL_LIST_FILE=$2
1111
# MD_NUM: PR中MD文件改动数量，用于判断是否需要进行文档超链接检测，默认为空，设置示例：export MD_NUM=10
1212
# WITHOUT_MD_NUM: PR中MD文件改动之外的改动文件数量，用于判断进行文档超链接检测后是否进行正常的CI，默认为空，设置示例：export WITHOUT_MD_NUM=10
1313

14-
# set -x
1514
if [[ $MODE == 'PaddleX' ]];then
15+
set -x
1616
failed_cmd_list=""
1717
fi
1818

@@ -52,25 +52,25 @@ function run_command(){
5252
printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}"
5353
eval $command
5454
last_status=${PIPESTATUS[0]}
55+
n=1
56+
# Try 2 times to run command if it fails
5557
if [[ $MODE != 'PaddleX' ]];then
56-
n=1
57-
# Try 3 times to run command if it fails
5858
while [[ $last_status != 0 ]]; do
5959
sleep 10
6060
n=`expr $n + 1`
6161
printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}"
62+
sync
63+
echo 1 > /proc/sys/vm/drop_caches
6264
eval $command
6365
last_status=${PIPESTATUS[0]}
64-
if [[ $n -eq 3 && $last_status != 0 ]]; then
65-
echo "Retry 3 times failed with command: ${command}"
66+
if [[ $n -eq 2 && $last_status != 0 ]]; then
67+
echo "Retry 2 times failed with command: ${command}"
6668
exit 1
6769
fi
6870
done
69-
else
70-
if [[ $last_status != 0 ]];then
71-
failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}"
72-
echo "Run ${command} failed"
73-
fi
71+
elif [[ $last_status != 0 ]]; then
72+
failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}"
73+
echo "Run ${command} failed"
7474
fi
7575
}
7676

@@ -188,7 +188,9 @@ function run_models(){
188188
checker_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --check --$check_option --output ${model_output_path} --check_weights_items ${check_weights_items} --module_name ${module_name}"
189189
run_command ${checker_cmd} ${module_name}
190190
done
191-
rm -rf ${model_output_path}/*[0-9]*
191+
if [[ $last_status -eq 0 ]];then
192+
rm -rf ${model_output_path}
193+
fi
192194
fi
193195
done
194196
model_dict[$module_name]="$model_list"
@@ -222,16 +224,19 @@ MODULE_OUTPUT_PATH=${BASE_PATH}/outputs
222224
CONFIG_FILE=${BASE_PATH}/config.txt
223225
pip config set global.index-url https://mirrors.bfsu.edu.cn/pypi/web/simple
224226
pip install beautifulsoup4==4.12.3
227+
pip install tqdm
225228
pip install markdown
226229
declare -A weight_dict
227230
declare -A model_dict
228231

229232
#################################################### 代码风格检查 ######################################################
230-
pre-commit
231-
last_status=${PIPESTATUS[0]}
232-
if [[ $last_status != 0 ]]; then
233-
echo "pre-commit check failed, please fix it first."
234-
exit 1
233+
if [[ DEVICE_TYPE == 'gpu' ]];then
234+
pre-commit
235+
last_status=${PIPESTATUS[0]}
236+
if [[ $last_status != 0 ]]; then
237+
echo "pre-commit check failed, please fix it first."
238+
exit 1
239+
fi
235240
fi
236241

237242
#################################################### 文档超链接检查 ######################################################
@@ -256,9 +261,16 @@ eval $install_pdx_cmd
256261
if [[ -z $MEM_SIZE ]]; then
257262
MEM_SIZE=16
258263
fi
264+
259265
if [[ -z $DEVICE_TYPE ]]; then
260266
DEVICE_TYPE='gpu'
261267
fi
268+
269+
PIPE_TYPE=$DEVICE_TYPE
270+
if [[ $DEVICE_TYPE == 'dcu' ]]; then
271+
DEVICE_TYPE='gpu'
272+
fi
273+
262274
if [[ -z $DEVICE_ID ]]; then
263275
DEVICE_ID='0,1,2,3'
264276
fi
@@ -380,6 +392,9 @@ for modules_info in ${modules_info_list[@]}; do
380392
done
381393
done
382394

395+
if [[ $PIPE_TYPE != 'gpu' ]];then
396+
exit 0
397+
fi
383398
#################################################### 产线级测试 ######################################################
384399
IFS=$'\n'
385400
PIPELINE_YAML_LIST=`ls paddlex/pipelines | grep .yaml`

‎models/PaddleX/ci/config.txt

Lines changed: 53 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ inference_weight_dir:best_accuracy/inference
4747
epochs_iters:2
4848
*********************************************************************************************************************
4949
suite_name:PaddleOCR
50-
module_name:text_detection_seal
51-
check_dataset_yaml:paddlex/configs/text_detection_seal/PP-OCRv4_mobile_seal_det.yaml
50+
module_name:seal_text_detection
51+
check_dataset_yaml:paddlex/configs/seal_text_detection/PP-OCRv4_mobile_seal_det.yaml
5252
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/ocr_curve_det_dataset_examples.tar
5353
train_list_name:train.txt
5454
run_model:train|evaluate|predict|export
@@ -71,13 +71,25 @@ inference_weight_dir:best_model/inference
7171
epochs_iters:2
7272
*********************************************************************************************************************
7373
suite_name:PaddleDetection
74+
module_name:rotated_object_detection
75+
check_dataset_yaml:paddlex/configs/rotated_object_detection/PP-YOLOE-R_L.yaml
76+
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/rdet_dota_examples.tar
77+
train_list_name:
78+
run_model:train|evaluate|predict|export
79+
check_options:check_train_result_json|check_eval_result_json
80+
check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info
81+
evaluate_weight_path:best_model/best_model.pdparams
82+
inference_weight_dir:best_model/inference
83+
epochs_iters:2
84+
*********************************************************************************************************************
85+
suite_name:PaddleDetection
7486
module_name:face_detection
7587
check_dataset_yaml:paddlex/configs/face_detection/PicoDet_LCNet_x2_5_face.yaml
7688
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/widerface_coco_examples.tar
7789
train_list_name:
7890
run_model:train|evaluate|predict|export
7991
check_options:check_train_result_json|check_eval_result_json
80-
check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info
92+
check_weights_items:2,pdparams,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info
8193
evaluate_weight_path:best_model/best_model.pdparams
8294
inference_weight_dir:best_model/inference
8395
epochs_iters:2
@@ -143,8 +155,8 @@ inference_weight_dir:best_model/inference
143155
epochs_iters:2
144156
*********************************************************************************************************************
145157
suite_name:PaddleDetection
146-
module_name:structure_analysis
147-
check_dataset_yaml:paddlex/configs/structure_analysis/PicoDet_layout_1x.yaml
158+
module_name:layout_detection
159+
check_dataset_yaml:paddlex/configs/layout_detection/PicoDet_layout_1x.yaml
148160
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/det_layout_examples.tar
149161
train_list_name:
150162
run_model:train|evaluate|predict|export
@@ -155,8 +167,8 @@ inference_weight_dir:best_model/inference
155167
epochs_iters:2
156168
*********************************************************************************************************************
157169
suite_name:PaddleClas
158-
module_name:face_recognition
159-
check_dataset_yaml:paddlex/configs/face_recognition/MobileFaceNet.yaml
170+
module_name:face_feature
171+
check_dataset_yaml:paddlex/configs/face_feature/MobileFaceNet.yaml
160172
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/face_rec_examples.tar
161173
train_list_name:train.txt
162174
run_model:train|evaluate|predict|export
@@ -191,8 +203,8 @@ inference_weight_dir:best_model/inference
191203
epochs_iters:2
192204
*********************************************************************************************************************
193205
suite_name:PaddleClas
194-
module_name:multilabel_classification
195-
check_dataset_yaml:paddlex/configs/multilabel_classification/ResNet50_ML.yaml
206+
module_name:image_multilabel_classification
207+
check_dataset_yaml:paddlex/configs/image_multilabel_classification/ResNet50_ML.yaml
196208
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mlcls_nus_examples.tar
197209
train_list_name:train.txt
198210
run_model:train|evaluate|predict|export
@@ -203,8 +215,8 @@ inference_weight_dir:best_model/inference
203215
epochs_iters:2
204216
*********************************************************************************************************************
205217
suite_name:PaddleClas
206-
module_name:pedestrian_attribute
207-
check_dataset_yaml:paddlex/configs/pedestrian_attribute/PP-LCNet_x1_0_pedestrian_attribute.yaml
218+
module_name:pedestrian_attribute_recognition
219+
check_dataset_yaml:paddlex/configs/pedestrian_attribute_recognition/PP-LCNet_x1_0_pedestrian_attribute.yaml
208220
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/pedestrian_attribute_examples.tar
209221
train_list_name:train.txt
210222
run_model:train|evaluate|predict|export
@@ -215,8 +227,8 @@ inference_weight_dir:best_model/inference
215227
epochs_iters:2
216228
*********************************************************************************************************************
217229
suite_name:PaddleClas
218-
module_name:vehicle_attribute
219-
check_dataset_yaml:paddlex/configs/vehicle_attribute/PP-LCNet_x1_0_vehicle_attribute.yaml
230+
module_name:vehicle_attribute_recognition
231+
check_dataset_yaml:paddlex/configs/vehicle_attribute_recognition/PP-LCNet_x1_0_vehicle_attribute.yaml
220232
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/vehicle_attribute_examples.tar
221233
train_list_name:train.txt
222234
run_model:train|evaluate|predict|export
@@ -227,8 +239,20 @@ inference_weight_dir:best_model/inference
227239
epochs_iters:2
228240
*********************************************************************************************************************
229241
suite_name:PaddleClas
230-
module_name:general_recognition
231-
check_dataset_yaml:paddlex/configs/general_recognition/PP-ShiTuV2_rec.yaml
242+
module_name:textline_orientation
243+
check_dataset_yaml:paddlex/configs/textline_orientation/PP-LCNet_x0_25_textline_ori.yaml
244+
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/textline_orientation_example_data.tar
245+
train_list_name:train.txt
246+
run_model:train|evaluate|predict|export
247+
check_options:check_train_result_json|check_eval_result_json
248+
check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info
249+
evaluate_weight_path:best_model/best_model.pdparams
250+
inference_weight_dir:best_model/inference
251+
epochs_iters:2
252+
*********************************************************************************************************************
253+
suite_name:PaddleClas
254+
module_name:image_feature
255+
check_dataset_yaml:paddlex/configs/image_feature/PP-ShiTuV2_rec.yaml
232256
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/Inshop_examples.tar
233257
train_list_name:train.txt
234258
run_model:train|evaluate|predict|export
@@ -263,8 +287,8 @@ inference_weight_dir:best_model/inference
263287
epochs_iters:20
264288
*********************************************************************************************************************
265289
suite_name:PaddleSeg
266-
module_name:anomaly_detection
267-
check_dataset_yaml:paddlex/configs/anomaly_detection/STFPM.yaml
290+
module_name:image_anomaly_detection
291+
check_dataset_yaml:paddlex/configs/image_anomaly_detection/STFPM.yaml
268292
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mvtec_examples.tar
269293
train_list_name:train.txt
270294
run_model:train|evaluate|predict|export
@@ -309,3 +333,15 @@ check_weights_items:score,pdparams
309333
evaluate_weight_path:best_model/model.pdparams
310334
inference_weight_dir:inference
311335
epochs_iters:2
336+
*********************************************************************************************************************
337+
suite_name:PaddleVideo
338+
module_name:video_classification
339+
check_dataset_yaml:paddlex/configs/video_classification/PPTSM_ResNet50_k400_8frames_uniform.yaml
340+
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/k400_examples.tar
341+
train_list_name:train.txt
342+
run_model:train|evaluate|predict|export
343+
check_options:check_train_result_json|check_eval_result_json
344+
check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info
345+
evaluate_weight_path:best_model/best_model.pdparams
346+
inference_weight_dir:best_model/inference
347+
epochs_iters:2

0 commit comments

Comments
 (0)