Only use act if fused_act is True

Author: Julius Berner

physicsnemo/models/diffusion/layers.py

@@ -354,7 +354,7 @@ def __init__(
         self.act_fn = None
         self.amp_mode = amp_mode
         if self.use_apex_gn:
-            if self.act:
+            if self.fused_act:
                 self.gn = ApexGroupNorm(
                     num_groups=self.num_groups,
                     num_channels=num_channels,