Add SkyAR module (#1218)

Author: jm12138

modules/thirdparty/video/Video_editing/SkyAR/README.md

@@ -0,0 +1,109 @@
+## 模型概述
+* SkyAR 是一种用于视频中天空置换与协调的视觉方法，该方法能够在风格可控的视频中自动生成逼真的天空背景。
+* 该算法是一种完全基于视觉的解决方案，它的好处就是可以处理非静态图像，同时不受拍摄设备的限制，也不需要用户交互，可以处理在线或离线视频。
+* 算法主要由三个核心组成：
+  * 天空抠图网络（Sky Matting Network）：就是一种 Matting 图像分隔，用于检测视频帧中天空区域的视频，可以精确地获得天空蒙版。
+  * 运动估计（Motion Estimation）：恢复天空运动的运动估计器，使生成的天空与摄像机的运动同步。
+  * 图像融合（Image Blending）：将用户指定的天空模板混合到视频帧中。除此之外，还用于重置和着色，使混合结果在其颜色和动态范围内更具视觉逼真感。
+* 整体框架图如下：
+
+	![](http://p4.itc.cn/q_70/images03/20201114/42eaf00af8dd4aa4ae3c0cdc6e50b793.jpeg)
+* 参考论文：Zhengxia Zou. [Castle in the Sky: Dynamic Sky Replacement and Harmonization in Videos](https://arxiv.org/abs/2010.11800). CoRR, abs/2010.118003, 2020.
+* 官方开源项目： [jiupinjia/SkyAR](https://github.com/jiupinjia/SkyAR)
+## 模型安装
+```shell
+$hub install SkyAR
+```
+
+## 效果展示
+* 原始视频：
+
+	![原始视频](https://img-blog.csdnimg.cn/20210126142046572.gif)
+
+* 木星：
+
+	![木星](https://img-blog.csdnimg.cn/20210125211435619.gif)
+* 雨天：
+
+	![雨天](https://img-blog.csdnimg.cn/2021012521152492.gif)
+* 银河：
+
+	![银河](https://img-blog.csdnimg.cn/20210125211523491.gif)
+* 第九区飞船：
+
+	![第九区飞船](https://img-blog.csdnimg.cn/20210125211520955.gif)
+* 原始视频：
+
+	![原始视频](https://img-blog.csdnimg.cn/20210126142038716.gif)
+* 漂浮城堡：
+
+	![漂浮城堡](https://img-blog.csdnimg.cn/20210125211514997.gif)
+* 电闪雷鸣：
+
+	![电闪雷鸣](https://img-blog.csdnimg.cn/20210125211433591.gif)
+* 超级月亮：
+
+	![超级月亮](https://img-blog.csdnimg.cn/20210125211417524.gif)
+
+## API 说明
+
+```python
+def MagicSky(
+        video_path, save_path, config='jupiter',
+        is_rainy=False, preview_frames_num=0, is_video_sky=False, is_show=False,
+        skybox_img=None, skybox_video=None, rain_cap_path=None,
+        halo_effect=True, auto_light_matching=False,
+        relighting_factor=0.8, recoloring_factor=0.5, skybox_center_crop=0.5
+    )
+```
+
+深度估计API
+
+**参数**
+
+* video_path(str)：输入视频路径
+* save_path(str)：视频保存路径
+* config(str): 预设 SkyBox 配置，所有预设配置如下，如果使用自定义 SkyBox，请设置为 None：
+```
+[
+    'cloudy', 'district9ship', 'floatingcastle', 'galaxy', 'jupiter',
+    'rainy', 'sunny', 'sunset', 'supermoon', 'thunderstorm'
+]
+```
+* skybox_img(str)：自定义的 SkyBox 图像路径
+* skybox_video(str)：自定义的 SkyBox 视频路径
+* is_video_sky(bool)：自定义 SkyBox 是否为视频
+* rain_cap_path(str)：自定义下雨效果视频路径
+* is_rainy(bool): 天空是否下雨
+* halo_effect(bool)：是否开启 halo effect
+* auto_light_matching(bool)：是否开启自动亮度匹配
+* relighting_factor(float): Relighting factor
+* recoloring_factor(float): Recoloring factor
+* skybox_center_crop(float)：SkyBox center crop factor
+* preview_frames_num(int)：设置预览帧数量，即只处理开头这几帧，设为 0，则为全部处理
+* is_show(bool)：是否图形化预览
+
+## 预测代码示例
+
+```python
+import paddlehub as hub
+
+model = hub.Module(name='SkyAR')
+
+model.MagicSky(
+    video_path=[path to input video path],
+    save_path=[path to save video path]
+)
+```
+
+## 模型相关信息
+
+### 模型代码
+
+https://github.com/jm12138/SkyAR_Paddle_GUI
+
+### 依赖
+
+paddlepaddle >= 2.0.0rc0
+
+paddlehub >= 2.0.0rc0

modules/thirdparty/video/Video_editing/SkyAR/__init__.py

@@ -0,0 +1 @@
+

modules/thirdparty/video/Video_editing/SkyAR/module.py

@@ -0,0 +1,73 @@
+import os
+import paddle.nn as nn
+from .skyfilter import SkyFilter
+from paddlehub.module.module import moduleinfo
+
+
+@moduleinfo(
+    name="SkyAR",
+    type="CV/Video_editing",
+    author="jm12138",
+    author_email="",
+    summary="SkyAR",
+    version="1.0.0"
+)
+class SkyAR(nn.Layer):
+    def __init__(self, model_path=None):
+        super(SkyAR, self).__init__()
+        self.imgs = ['cloudy', 'district9ship', 'floatingcastle',
+                     'galaxy', 'jupiter', 'rainy', 'sunny', 'sunset', 'supermoon']
+        self.videos = ['thunderstorm']
+        if model_path:
+            self.model_path = model_path
+        else:
+            self.model_path = os.path.join(self.directory, './ResNet50FCN')
+
+    def MagicSky(
+        self, video_path, save_path, config='jupiter',
+        is_rainy=False, preview_frames_num=0, is_video_sky=False, is_show=False,
+        skybox_img=None, skybox_video=None, rain_cap_path=None,
+        halo_effect=True, auto_light_matching=False,
+        relighting_factor=0.8, recoloring_factor=0.5, skybox_center_crop=0.5
+    ):
+        if config in self.imgs:
+            skybox_img = os.path.join(
+                self.directory, 'skybox', '%s.jpg' % config)
+            skybox_video = None
+            is_video_sky = False
+        elif config in self.videos:
+            skybox_img = None
+            skybox_video = os.path.join(
+                self.directory, 'skybox', '%s.mp4' % config)
+            is_video_sky = True
+        elif skybox_img:
+            is_video_sky = False
+            skybox_video = None
+        elif is_video_sky and skybox_video:
+            skybox_img = None
+        else:
+            raise 'please check your configs'
+
+        if not rain_cap_path:
+            rain_cap_path = os.path.join(
+                self.directory, 'rain_streaks', 'videoplayback.mp4')
+
+        skyfilter = SkyFilter(
+            model_path=self.model_path,
+            video_path=video_path,
+            save_path=save_path,
+            in_size=(384, 384),
+            halo_effect=halo_effect,
+            auto_light_matching=auto_light_matching,
+            relighting_factor=relighting_factor,
+            recoloring_factor=recoloring_factor,
+            skybox_center_crop=skybox_center_crop,
+            rain_cap_path=rain_cap_path,
+            skybox_img=skybox_img,
+            skybox_video=skybox_video,
+            is_video=is_video_sky,
+            is_rainy=is_rainy,
+            is_show=is_show
+        )
+
+        skyfilter.run(preview_frames_num)

modules/thirdparty/video/Video_editing/SkyAR/rain.py

@@ -0,0 +1,62 @@
+import cv2
+import numpy as np
+
+__all__ = ['Rain']
+
+
+class Rain():
+    def __init__(self, rain_cap_path, rain_intensity=1.0, haze_intensity=4.0, gamma=2.0, light_correction=0.9):
+        self.rain_intensity = rain_intensity
+        self.haze_intensity = haze_intensity
+        self.gamma = gamma
+        self.light_correction = light_correction
+        self.frame_id = 1
+
+        self.cap = cv2.VideoCapture(rain_cap_path)
+
+    def _get_rain_layer(self):
+        ret, frame = self.cap.read()
+        if ret:
+            rain_layer = frame
+        else:  # if reach the last frame, read from the begining
+            self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
+            ret, frame = self.cap.read()
+            rain_layer = frame
+
+        rain_layer = cv2.cvtColor(rain_layer, cv2.COLOR_BGR2RGB) / 255.0
+        rain_layer = np.array(rain_layer, dtype=np.float32)
+
+        return rain_layer
+
+    def _create_haze_layer(self, rain_layer):
+        return 0.1*np.ones_like(rain_layer)
+
+    def forward(self, img):
+        # get input image size
+        h, w, c = img.shape
+
+        # create a rain layer
+        rain_layer = self._get_rain_layer()
+
+        rain_layer = cv2.resize(rain_layer, (w, h))
+        rain_layer = cv2.blur(rain_layer, (3, 3))
+        rain_layer = rain_layer * \
+            (1 - cv2.boxFilter(img, -1, (int(w/10), int(h/10))))
+
+        # create a haze layer
+        haze_layer = self._create_haze_layer(rain_layer)
+
+        # combine the rain layer and haze layer together
+        rain_layer = self.rain_intensity*rain_layer + \
+            self.haze_intensity*haze_layer
+
+        # synthesize an output image (screen blend)
+        img_out = 1 - (1 - rain_layer) * (1 - img)
+
+        # gamma and light correction
+        img_out = self.light_correction*(img_out**self.gamma)
+
+        # check boundary
+        img_out = np.clip(img_out, a_min=0, a_max=1.)
+
+        return img_out