Skip to content

Commit 4b91d6f

Browse files
authored
convert : only check for tokenizer folder if we need it (#14704)
1 parent cf91f21 commit 4b91d6f

File tree

1 file changed

+6
-5
lines changed

1 file changed

+6
-5
lines changed

convert_hf_to_gguf_update.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -240,11 +240,6 @@ def get_existing_models(convert_py):
240240
if tokt == TOKENIZER_TYPE.SPM or tokt == TOKENIZER_TYPE.UGM:
241241
continue
242242

243-
# Skip if the tokenizer folder does not exist or there are other download issues previously
244-
if not os.path.exists(f"models/tokenizers/{name}"):
245-
logger.warning(f"Directory for tokenizer {name} not found. Skipping...")
246-
continue
247-
248243
# create the tokenizer
249244
if chkhsh is not None:
250245
# if the model has a pre-computed hash, use it
@@ -254,6 +249,12 @@ def get_existing_models(convert_py):
254249
chkhsh = existing_models[name]
255250
else:
256251
# otherwise, compute the hash of the tokenizer
252+
253+
# Skip if the tokenizer folder does not exist or there are other download issues previously
254+
if not os.path.exists(f"models/tokenizers/{name}"):
255+
logger.warning(f"Directory for tokenizer {name} not found. Skipping...")
256+
continue
257+
257258
try:
258259
logger.info(f"Loading tokenizer from {f'models/tokenizers/{name}'}...")
259260
if name == "t5":

0 commit comments

Comments
 (0)