Add npu support (#3717)

Author: Birdylx

paddleseg/models/layers/layer_libs.py

@@ -18,16 +18,133 @@
 import paddle.nn as nn
 import paddle.nn.functional as F
 from paddleseg.models import layers
+import paddle.distributed as dist
+import math
+
+
+class _AllReduce(paddle.autograd.PyLayer):
+
+    @staticmethod
+    def forward(ctx, input):
+        input_list = [
+            paddle.zeros_like(input) for k in range(dist.get_world_size())
+        ]
+        # Use allgather instead of allreduce since I don't trust in-place operations ..
+        dist.all_gather(input_list, input, sync_op=True)
+        inputs = paddle.stack(input_list, axis=0)
+        return paddle.sum(inputs, axis=0)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        dist.all_reduce(grad_output, sync_op=True)
+        return grad_output
+
+
+def differentiable_all_reduce(input):
+    """
+    Differentiable counterpart of `dist.all_reduce`.
+    """
+    if (not dist.is_available() or not dist.is_initialized()
+            or dist.get_world_size() == 1):
+        return input
+    return _AllReduce.apply(input)
+
+
+class NaiveSyncBatchNorm(nn.BatchNorm2D):
+
+    def __init__(self, *args, stats_mode="", **kwargs):
+        super().__init__(*args, **kwargs)
+        assert stats_mode in ["", "N"]
+        self._stats_mode = stats_mode
+
+    def forward(self, input):
+        if dist.get_world_size() == 1 or not self.training:
+            return super().forward(input)
+
+        B, C = input.shape[0], input.shape[1]
+
+        mean = paddle.mean(input, axis=[0, 2, 3])
+        meansqr = paddle.mean(input * input, axis=[0, 2, 3])
+
+        if self._stats_mode == "":
+            assert B > 0, 'SyncBatchNorm(stats_mode="") does not support zero batch size.'
+            vec = paddle.concat([mean, meansqr], axis=0)
+            vec = differentiable_all_reduce(vec) * (1.0 / dist.get_world_size())
+            mean, meansqr = paddle.split(vec, [C, C])
+            momentum = 1 - self._momentum  # NOTE: paddle has reverse momentum defination
+        else:
+            if B == 0:
+                vec = paddle.zeros([2 * C + 1], dtype=mean.dtype)
+                vec = vec + input.sum(
+                )  # make sure there is gradient w.r.t input
+            else:
+                vec = paddle.concat(
+                    [
+                        mean,
+                        meansqr,
+                        paddle.ones([1], dtype=mean.dtype),
+                    ],
+                    axis=0,
+                )
+            vec = differentiable_all_reduce(vec * B)
+
+            total_batch = vec[-1].detach()
+            momentum = total_batch.clip(max=1) * (
+                1 - self._momentum)  # no update if total_batch is 0
+            mean, meansqr, _ = paddle.split(
+                vec / total_batch.clip(min=1),
+                [C, C, int(vec.shape[0] - 2 * C)])  # avoid div-by-zero
+
+        var = meansqr - mean * mean
+        invstd = paddle.rsqrt(var + self._epsilon)
+        scale = self.weight * invstd
+        bias = self.bias - mean * scale
+        scale = scale.reshape([1, -1, 1, 1])
+        bias = bias.reshape([1, -1, 1, 1])
+
+        tmp_mean = self._mean + momentum * (mean.detach() - self._mean)
+        self._mean.set_value(tmp_mean)
+        tmp_variance = self._variance + (momentum *
+                                         (var.detach() - self._variance))
+        self._variance.set_value(tmp_variance)
+        ret = input * scale + bias
+        return ret
+
+    @classmethod
+    def convert_sync_batchnorm(cls, layer):
+        layer_output = layer
+        if isinstance(layer, nn.BatchNorm2D):
+
+            layer_output = NaiveSyncBatchNorm(layer._num_features,
+                                              layer._momentum, layer._epsilon,
+                                              layer._weight_attr,
+                                              layer._bias_attr,
+                                              layer._data_format, layer._name)
+
+            if (layer._weight_attr is not False
+                    and layer._bias_attr is not False):
+                with paddle.no_grad():
+                    layer_output.weight = layer.weight
+                    layer_output.bias = layer.bias
+            layer_output._mean = layer._mean
+            layer_output._variance = layer._variance
+
+        for name, sublayer in layer.named_children():
+            layer_output.add_sublayer(name,
+                                      cls.convert_sync_batchnorm(sublayer))
+        del layer
+        return layer_output
 
 
 def SyncBatchNorm(*args, **kwargs):
     """In cpu environment nn.SyncBatchNorm does not have kernel so use nn.BatchNorm2D instead"""
     if paddle.get_device() == 'cpu' or os.environ.get(
-            'PADDLESEG_EXPORT_STAGE') or 'xpu' in paddle.get_device(
-            ) or 'npu' in paddle.get_device():
+            'PADDLESEG_EXPORT_STAGE') or 'xpu' in paddle.get_device():
         return nn.BatchNorm2D(*args, **kwargs)
     elif paddle.distributed.ParallelEnv().nranks == 1:
         return nn.BatchNorm2D(*args, **kwargs)
+    elif 'npu' in paddle.get_device():
+        return NaiveSyncBatchNorm(*args, **kwargs)
     else:
         return nn.SyncBatchNorm(*args, **kwargs)
 

paddleseg/models/layers/pyramid_pool.py

@@ -19,6 +19,15 @@
 from paddleseg.models import layers
 
 
+class CustomAvgPool2D(nn.Layer):
+
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x):
+        return paddle.mean(x, axis=[2, 3], keepdim=True)
+
+
 class ASPPModule(nn.Layer):
     """
     Atrous Spatial Pyramid Pooling.
@@ -64,9 +73,13 @@ def __init__(self,
         out_size = len(self.aspp_blocks)
 
         if image_pooling:
+            # avgpool with outsize=(1,1) is extreamly slow when backward,
+            # so we replace it with equivalent mean operation.
+            pool_layer = CustomAvgPool2D() if "npu" in paddle.get_device(
+            ) else nn.AdaptiveAvgPool2D(output_size=(1, 1),
+                                        data_format=data_format)
             self.global_avg_pool = nn.Sequential(
-                nn.AdaptiveAvgPool2D(output_size=(1, 1),
-                                     data_format=data_format),
+                pool_layer,
                 layers.ConvBNReLU(in_channels,
                                   out_channels,
                                   kernel_size=1,

paddleseg/utils/utils.py

@@ -25,6 +25,7 @@
 
 from paddleseg.utils import logger, seg_env, get_sys_env
 from paddleseg.utils.download import download_file_and_uncompress
+from paddleseg.models.layers.layer_libs import NaiveSyncBatchNorm
 
 
 def set_seed(seed=None):
@@ -83,6 +84,9 @@ def convert_sync_batchnorm(model, device):
         and env_info['GPUs used'] and paddle.distributed.ParallelEnv().nranks > 1:
         model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model)
         logger.info("Convert bn to sync_bn")
+    elif device == "npu" and paddle.distributed.ParallelEnv().nranks > 1:
+        model = NaiveSyncBatchNorm.convert_sync_batchnorm(model)
+        logger.info("Convert bn to sync_bn in NPU Device")
     return model
 
 
@@ -97,7 +101,7 @@ def set_cv2_num_threads(num_workers):
 
 
 @contextlib.contextmanager
-def generate_tempdir(directory: str=None, **kwargs):
+def generate_tempdir(directory: str = None, **kwargs):
     '''Generate a temporary directory'''
     directory = seg_env.TMP_HOME if not directory else directory
     with tempfile.TemporaryDirectory(dir=directory, **kwargs) as _dir:
@@ -160,23 +164,24 @@ def load_pretrained_model(model, pretrained_model):
             for k in keys:
                 if k not in para_state_dict:
                     logger.warning("{} is not in pretrained model".format(k))
-                elif list(para_state_dict[k].shape) != list(model_state_dict[k]
-                                                            .shape):
+                elif list(para_state_dict[k].shape) != list(
+                        model_state_dict[k].shape):
                     logger.warning(
                         "[SKIP] Shape of pretrained params {} doesn't match.(Pretrained: {}, Actual: {})"
-                        .format(k, para_state_dict[k].shape, model_state_dict[k]
-                                .shape))
+                        .format(k, para_state_dict[k].shape,
+                                model_state_dict[k].shape))
                 else:
                     model_state_dict[k] = para_state_dict[k]
                     num_params_loaded += 1
             model.set_dict(model_state_dict)
             logger.info("There are {}/{} variables loaded into {}.".format(
-                num_params_loaded,
-                len(model_state_dict), model.__class__.__name__))
+                num_params_loaded, len(model_state_dict),
+                model.__class__.__name__))
 
         else:
-            raise ValueError('The pretrained model directory is not Found: {}'.
-                             format(pretrained_model))
+            raise ValueError(
+                'The pretrained model directory is not Found: {}'.format(
+                    pretrained_model))
     else:
         logger.info(
             'No pretrained model to load, {} will be trained from scratch.'.
@@ -251,6 +256,7 @@ def get_image_list(image_path):
 
 
 class NoAliasDumper(yaml.SafeDumper):
+
     def ignore_aliases(self, data):
         return True