Skip to content

Commit 2e64244

Browse files
Add ernie-tiny configs to ERNIE tokenizer (PaddlePaddle#406)
Add ernie-tiny configs to ERNIE tokenizer Co-authored-by: Guo Sheng <whucsgs@163.com>
1 parent 55e96ba commit 2e64244

File tree

1 file changed

+5
-0
lines changed

1 file changed

+5
-0
lines changed

paddlenlp/transformers/ernie/tokenizer.py

+5
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ class ErnieTokenizer(PretrainedTokenizer):
7272
"vocab_file": {
7373
"ernie-1.0":
7474
"https://paddlenlp.bj.bcebos.com/models/transformers/ernie/vocab.txt",
75+
"ernie-tiny":
76+
"https://paddlenlp.bj.bcebos.com/models/transformers/ernie_tiny/vocab.txt",
7577
"ernie-2.0-en":
7678
"https://paddlenlp.bj.bcebos.com/models/transformers/ernie_v2_base/vocab.txt",
7779
"ernie-2.0-large-en":
@@ -88,6 +90,9 @@ class ErnieTokenizer(PretrainedTokenizer):
8890
"ernie-1.0": {
8991
"do_lower_case": True
9092
},
93+
"ernie-tiny": {
94+
"do_lower_case": True
95+
},
9196
"ernie-2.0-en": {
9297
"do_lower_case": True
9398
},

0 commit comments

Comments
 (0)