Skip to content

Commit 32787fc

Browse files
changdazhouchangdazhou
committed
update paddlex ci
1 parent 874bb1f commit 32787fc

File tree

4 files changed

+67
-62
lines changed

4 files changed

+67
-62
lines changed

‎models/PaddleX/ci/black_list.txt

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,11 @@
11
################################### All ##################################
2-
All:PP-YOLOE_plus_SOD-largesize-L
32
################################## Train #################################
43
################################ Evaluate ################################
54
################################# Predict ################################
6-
Predict:PP-YOLOE_seg-S
7-
Predict:MaskRCNN-ResNet50
8-
Predict:MaskRCNN-ResNet50-FPN
9-
Predict:MaskRCNN-ResNet50-vd-FPN
10-
Predict:MaskRCNN-ResNet101-FPN
11-
Predict:MaskRCNN-ResNet101-vd-FPN
12-
Predict:MaskRCNN-ResNeXt101-vd-FPN
13-
Predict:Cascade-MaskRCNN-ResNet50-vd-SSLDv2-FPN
14-
Predict:Cascade-MaskRCNN-ResNet50-FPN
15-
Predict:Nonstationary_ad
165
Predict:TimesNet_ad
17-
Predict:PatchTST_ad
18-
Predict:DLinear_ad
19-
Predict:PP-YOLOE_plus_SOD-L
20-
Predict:PP-YOLOE_plus_SOD-S
21-
Predict:Cascade-FasterRCNN-ResNet50-vd-SSLDv2-FPN
22-
Predict:Cascade-FasterRCNN-ResNet50-FPN
23-
Predict:FasterRCNN-Swin-Tiny-FPN
24-
Predict:FasterRCNN-ResNeXt101-vd-FPN
25-
Predict:FasterRCNN-ResNet101-FPN
26-
Predict:FasterRCNN-ResNet101
27-
Predict:FasterRCNN-ResNet50-vd-SSLDv2-FPN
28-
Predict:FasterRCNN-ResNet50-vd-FPN
29-
Predict:FasterRCNN-ResNet50-FPN
30-
Predict:FasterRCNN-ResNet50
31-
Predict:FasterRCNN-ResNet34-FPN
326
################################## Export ################################
337
################################# Pipeline ###############################
348
Pipeline:PP-ChatOCRv3-doc
359
Pipeline:PP-ShiTuV2
3610
Pipeline:face_recognition
11+
Pipeline:ts_ad

‎models/PaddleX/ci/checker.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -227,8 +227,8 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso
227227
pass_flag = False
228228
else:
229229
show_type = json_data.get("show_type")
230-
if show_type not in ["image", "txt", "csv"]:
231-
check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv'其中一个")
230+
if show_type not in ["image", "txt", "csv", "video"]:
231+
check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv','video'其中一个")
232232
pass_flag = False
233233

234234
if module_name == "general_recognition":
@@ -249,6 +249,7 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso
249249
or "table" in module_name
250250
or "formula" in module_name
251251
or module_name == "general_recognition"
252+
or module_name == "face_feature"
252253
):
253254
try:
254255
num_class = int(json_data["attributes"].get("num_classes"))
@@ -368,7 +369,7 @@ def run_checks(self, args):
368369

369370
if args.check_train_result_json:
370371
# 检查 train_result.json 内容
371-
train_result_json = os.path.join(output_dir, "train_results.json")
372+
train_result_json = os.path.join(output_dir, "train_result.json")
372373
check_weights_items = args.check_weights_items
373374
check_train_json_message = []
374375
check_train_json_flag, check_train_json_message = self.check_train_json_content(

‎models/PaddleX/ci/ci_run.sh

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ MODEL_LIST_FILE=$2
1111
# MD_NUM: PR中MD文件改动数量，用于判断是否需要进行文档超链接检测，默认为空，设置示例：export MD_NUM=10
1212
# WITHOUT_MD_NUM: PR中MD文件改动之外的改动文件数量，用于判断进行文档超链接检测后是否进行正常的CI，默认为空，设置示例：export WITHOUT_MD_NUM=10
1313

14-
# set -x
1514
if [[ $MODE == 'PaddleX' ]];then
15+
set -x
1616
failed_cmd_list=""
1717
fi
1818

@@ -52,25 +52,25 @@ function run_command(){
5252
printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}"
5353
eval $command
5454
last_status=${PIPESTATUS[0]}
55+
n=1
56+
# Try 2 times to run command if it fails
5557
if [[ $MODE != 'PaddleX' ]];then
56-
n=1
57-
# Try 3 times to run command if it fails
5858
while [[ $last_status != 0 ]]; do
5959
sleep 10
6060
n=`expr $n + 1`
6161
printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}"
62+
sync
63+
echo 1 > /proc/sys/vm/drop_caches
6264
eval $command
6365
last_status=${PIPESTATUS[0]}
64-
if [[ $n -eq 3 && $last_status != 0 ]]; then
65-
echo "Retry 3 times failed with command: ${command}"
66+
if [[ $n -eq 2 && $last_status != 0 ]]; then
67+
echo "Retry 2 times failed with command: ${command}"
6668
exit 1
6769
fi
6870
done
69-
else
70-
if [[ $last_status != 0 ]];then
71-
failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}"
72-
echo "Run ${command} failed"
73-
fi
71+
elif [[ $last_status != 0 ]]; then
72+
failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}"
73+
echo "Run ${command} failed"
7474
fi
7575
}
7676

@@ -188,7 +188,9 @@ function run_models(){
188188
checker_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --check --$check_option --output ${model_output_path} --check_weights_items ${check_weights_items} --module_name ${module_name}"
189189
run_command ${checker_cmd} ${module_name}
190190
done
191-
rm -rf ${model_output_path}/*[0-9]*
191+
if [[ $last_status -eq 0 ]];then
192+
rm -rf ${model_output_path}
193+
fi
192194
fi
193195
done
194196
model_dict[$module_name]="$model_list"
@@ -222,16 +224,19 @@ MODULE_OUTPUT_PATH=${BASE_PATH}/outputs
222224
CONFIG_FILE=${BASE_PATH}/config.txt
223225
pip config set global.index-url https://mirrors.bfsu.edu.cn/pypi/web/simple
224226
pip install beautifulsoup4==4.12.3
227+
pip install tqdm
225228
pip install markdown
226229
declare -A weight_dict
227230
declare -A model_dict
228231

229232
#################################################### 代码风格检查 ######################################################
230-
pre-commit
231-
last_status=${PIPESTATUS[0]}
232-
if [[ $last_status != 0 ]]; then
233-
echo "pre-commit check failed, please fix it first."
234-
exit 1
233+
if [[ DEVICE_TYPE == 'gpu' ]];then
234+
pre-commit
235+
last_status=${PIPESTATUS[0]}
236+
if [[ $last_status != 0 ]]; then
237+
echo "pre-commit check failed, please fix it first."
238+
exit 1
239+
fi
235240
fi
236241

237242
#################################################### 文档超链接检查 ######################################################

‎models/PaddleX/ci/config.txt

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ inference_weight_dir:best_accuracy/inference
4747
epochs_iters:2
4848
*********************************************************************************************************************
4949
suite_name:PaddleOCR
50-
module_name:text_detection_seal
51-
check_dataset_yaml:paddlex/configs/text_detection_seal/PP-OCRv4_mobile_seal_det.yaml
50+
module_name:seal_text_detection
51+
check_dataset_yaml:paddlex/configs/seal_text_detection/PP-OCRv4_mobile_seal_det.yaml
5252
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/ocr_curve_det_dataset_examples.tar
5353
train_list_name:train.txt
5454
run_model:train|evaluate|predict|export
@@ -77,7 +77,7 @@ dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/widerface_co
7777
train_list_name:
7878
run_model:train|evaluate|predict|export
7979
check_options:check_train_result_json|check_eval_result_json
80-
check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info
80+
check_weights_items:2,pdparams,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info
8181
evaluate_weight_path:best_model/best_model.pdparams
8282
inference_weight_dir:best_model/inference
8383
epochs_iters:2
@@ -143,8 +143,8 @@ inference_weight_dir:best_model/inference
143143
epochs_iters:2
144144
*********************************************************************************************************************
145145
suite_name:PaddleDetection
146-
module_name:structure_analysis
147-
check_dataset_yaml:paddlex/configs/structure_analysis/PicoDet_layout_1x.yaml
146+
module_name:layout_detection
147+
check_dataset_yaml:paddlex/configs/layout_detection/PicoDet_layout_1x.yaml
148148
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/det_layout_examples.tar
149149
train_list_name:
150150
run_model:train|evaluate|predict|export
@@ -155,8 +155,8 @@ inference_weight_dir:best_model/inference
155155
epochs_iters:2
156156
*********************************************************************************************************************
157157
suite_name:PaddleClas
158-
module_name:face_recognition
159-
check_dataset_yaml:paddlex/configs/face_recognition/MobileFaceNet.yaml
158+
module_name:face_feature
159+
check_dataset_yaml:paddlex/configs/face_feature/MobileFaceNet.yaml
160160
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/face_rec_examples.tar
161161
train_list_name:train.txt
162162
run_model:train|evaluate|predict|export
@@ -191,8 +191,8 @@ inference_weight_dir:best_model/inference
191191
epochs_iters:2
192192
*********************************************************************************************************************
193193
suite_name:PaddleClas
194-
module_name:multilabel_classification
195-
check_dataset_yaml:paddlex/configs/multilabel_classification/ResNet50_ML.yaml
194+
module_name:image_multilabel_classification
195+
check_dataset_yaml:paddlex/configs/image_multilabel_classification/ResNet50_ML.yaml
196196
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mlcls_nus_examples.tar
197197
train_list_name:train.txt
198198
run_model:train|evaluate|predict|export
@@ -203,8 +203,8 @@ inference_weight_dir:best_model/inference
203203
epochs_iters:2
204204
*********************************************************************************************************************
205205
suite_name:PaddleClas
206-
module_name:pedestrian_attribute
207-
check_dataset_yaml:paddlex/configs/pedestrian_attribute/PP-LCNet_x1_0_pedestrian_attribute.yaml
206+
module_name:pedestrian_attribute_recognition
207+
check_dataset_yaml:paddlex/configs/pedestrian_attribute_recognition/PP-LCNet_x1_0_pedestrian_attribute.yaml
208208
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/pedestrian_attribute_examples.tar
209209
train_list_name:train.txt
210210
run_model:train|evaluate|predict|export
@@ -215,8 +215,8 @@ inference_weight_dir:best_model/inference
215215
epochs_iters:2
216216
*********************************************************************************************************************
217217
suite_name:PaddleClas
218-
module_name:vehicle_attribute
219-
check_dataset_yaml:paddlex/configs/vehicle_attribute/PP-LCNet_x1_0_vehicle_attribute.yaml
218+
module_name:vehicle_attribute_recognition
219+
check_dataset_yaml:paddlex/configs/vehicle_attribute_recognition/PP-LCNet_x1_0_vehicle_attribute.yaml
220220
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/vehicle_attribute_examples.tar
221221
train_list_name:train.txt
222222
run_model:train|evaluate|predict|export
@@ -227,8 +227,20 @@ inference_weight_dir:best_model/inference
227227
epochs_iters:2
228228
*********************************************************************************************************************
229229
suite_name:PaddleClas
230-
module_name:general_recognition
231-
check_dataset_yaml:paddlex/configs/general_recognition/PP-ShiTuV2_rec.yaml
230+
module_name:textline_orientation
231+
check_dataset_yaml:paddlex/configs/textline_orientation/PP-LCNet_x0_25_textline_ori.yaml
232+
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/textline_orientation_example_data.tar
233+
train_list_name:train.txt
234+
run_model:train|evaluate|predict|export
235+
check_options:check_train_result_json|check_eval_result_json
236+
check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info
237+
evaluate_weight_path:best_model/best_model.pdparams
238+
inference_weight_dir:best_model/inference
239+
epochs_iters:2
240+
*********************************************************************************************************************
241+
suite_name:PaddleClas
242+
module_name:image_feature
243+
check_dataset_yaml:paddlex/configs/image_feature/PP-ShiTuV2_rec.yaml
232244
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/Inshop_examples.tar
233245
train_list_name:train.txt
234246
run_model:train|evaluate|predict|export
@@ -263,8 +275,8 @@ inference_weight_dir:best_model/inference
263275
epochs_iters:20
264276
*********************************************************************************************************************
265277
suite_name:PaddleSeg
266-
module_name:anomaly_detection
267-
check_dataset_yaml:paddlex/configs/anomaly_detection/STFPM.yaml
278+
module_name:image_anomaly_detection
279+
check_dataset_yaml:paddlex/configs/image_anomaly_detection/STFPM.yaml
268280
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mvtec_examples.tar
269281
train_list_name:train.txt
270282
run_model:train|evaluate|predict|export
@@ -309,3 +321,15 @@ check_weights_items:score,pdparams
309321
evaluate_weight_path:best_model/model.pdparams
310322
inference_weight_dir:inference
311323
epochs_iters:2
324+
*********************************************************************************************************************
325+
suite_name:PaddleVideo
326+
module_name:video_classification
327+
check_dataset_yaml:paddlex/configs/video_classification/PPTSM_ResNet50_k400_8frames_uniform.yaml
328+
dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/k400_examples.tar
329+
train_list_name:train.txt
330+
run_model:train|evaluate|predict|export
331+
check_options:check_train_result_json|check_eval_result_json
332+
check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info
333+
evaluate_weight_path:best_model/best_model.pdparams
334+
inference_weight_dir:best_model/inference
335+
epochs_iters:2

0 commit comments

Comments
 (0)