From 2865cb730941627ee8ff2606dad9fd0d27e87914 Mon Sep 17 00:00:00 2001 From: Alex Butler Date: Wed, 23 Apr 2025 21:30:30 +0100 Subject: [PATCH 1/2] vae handle HIP OOM exceptions --- comfy/model_management.py | 8 ++++++++ comfy/sd.py | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 43e40224343..464947d5dfd 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -235,6 +235,14 @@ def mac_version(): except: OOM_EXCEPTION = Exception + +def is_oom_exception(ex): + if isinstance(ex, OOM_EXCEPTION): + return True + # handle also other kinds of oom, e.g. "HIP error: out of memory" + msg = str(ex) + return "out of memory" in msg + XFORMERS_VERSION = "" XFORMERS_ENABLED_VAE = True if args.disable_xformers: diff --git a/comfy/sd.py b/comfy/sd.py index 8aba5d65552..788bf9e7b80 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -592,8 +592,9 @@ def encode(self, pixel_samples): if samples is None: samples = torch.empty((pixel_samples.shape[0],) + tuple(out.shape[1:]), device=self.output_device) samples[x:x + batch_number] = out - - except model_management.OOM_EXCEPTION: + except Exception as ex: + if not model_management.is_oom_exception(ex): + raise logging.warning("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.") if self.latent_dim == 3: tile = 256 From 398805d794b400b74619ad857c6e52ed260fca0c Mon Sep 17 00:00:00 2001 From: Alex Butler Date: Wed, 23 Apr 2025 21:33:58 +0100 Subject: [PATCH 2/2] vae decode handle HIP oom exceptions --- comfy/sd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/comfy/sd.py b/comfy/sd.py index 788bf9e7b80..1b8bc2927aa 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -529,7 +529,9 @@ def decode(self, samples_in, vae_options={}): if pixel_samples is None: pixel_samples = torch.empty((samples_in.shape[0],) + tuple(out.shape[1:]), device=self.output_device) pixel_samples[x:x+batch_number] = out - except model_management.OOM_EXCEPTION: + except Exception as ex: + if not model_management.is_oom_exception(ex): + raise logging.warning("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.") dims = samples_in.ndim - 2 if dims == 1: