File tree Expand file tree Collapse file tree 1 file changed +6
-5
lines changed Expand file tree Collapse file tree 1 file changed +6
-5
lines changed Original file line number Diff line number Diff line change @@ -240,11 +240,6 @@ def get_existing_models(convert_py):
240
240
if tokt == TOKENIZER_TYPE .SPM or tokt == TOKENIZER_TYPE .UGM :
241
241
continue
242
242
243
- # Skip if the tokenizer folder does not exist or there are other download issues previously
244
- if not os .path .exists (f"models/tokenizers/{ name } " ):
245
- logger .warning (f"Directory for tokenizer { name } not found. Skipping..." )
246
- continue
247
-
248
243
# create the tokenizer
249
244
if chkhsh is not None :
250
245
# if the model has a pre-computed hash, use it
@@ -254,6 +249,12 @@ def get_existing_models(convert_py):
254
249
chkhsh = existing_models [name ]
255
250
else :
256
251
# otherwise, compute the hash of the tokenizer
252
+
253
+ # Skip if the tokenizer folder does not exist or there are other download issues previously
254
+ if not os .path .exists (f"models/tokenizers/{ name } " ):
255
+ logger .warning (f"Directory for tokenizer { name } not found. Skipping..." )
256
+ continue
257
+
257
258
try :
258
259
logger .info (f"Loading tokenizer from { f'models/tokenizers/{ name } ' } ..." )
259
260
if name == "t5" :
You can’t perform that action at this time.
0 commit comments