1.add compare result, 2.add seed for paddlegan (#493)

* 1.add compare result, 2.add seed for paddlegan

* 1.add compare result, 2.add seed for paddlegan

* 1.add compare result, 2.add seed for paddlegan

* 1.add compare result, 2.add seed for paddlegan

Author: lzzyzlbb

ppgan/datasets/firstorder_dataset.py

@@ -123,7 +123,7 @@ def read_video(name: Path, frame_shape=tuple([256, 256, 3]), saveto='folder'):
             except FileExistsError:
                 pass
             for idx, img in enumerate(video_array_reshape):
-                cv2.imwrite(sub_dir.joinpath('%i.png' % idx), img)
+                cv2.imwrite(str(sub_dir.joinpath('%i.png' % idx)), img[:,:,[2,1,0]])
             name.unlink()
         return video_array_reshape
     else:
@@ -207,7 +207,6 @@ def __getitem__(self, idx):
                     num_frames, replace=True,
                     size=2)) if self.is_train else range(num_frames)
             video_array = [video_array[i] for i in frame_idx]
-
         # convert to 3-channel image
         if video_array[0].shape[-1] == 4:
             video_array = [i[..., :3] for i in video_array]
@@ -218,7 +217,6 @@ def __getitem__(self, idx):
                 np.tile(i[..., np.newaxis], (1, 1, 3)) for i in video_array
             ]
         out = {}
-
         if self.is_train:
             if self.transform is not None:  #modify
                 t = self.transform(tuple(video_array))

ppgan/utils/options.py

@@ -67,6 +67,12 @@ def parse_args():
                         default=None,
                         help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
     )
+    # fix random numbers by setting seed
+    parser.add_argument('--seed',
+                        type=int,
+                        default=None,
+                        help='fix random numbers by setting seed\".'
+    )
     args = parser.parse_args()
 
     return args

ppgan/utils/setup.py

@@ -15,10 +15,10 @@
 import os
 import time
 import paddle
-
+import numpy as np
+import random
 from .logger import setup_logger
 
-
 def setup(args, cfg):
     if args.evaluate_only:
         cfg.is_train = False
@@ -44,3 +44,10 @@ def setup(args, cfg):
         paddle.set_device('gpu')
     else:
         paddle.set_device('cpu')
+    
+    if args.seed:
+        paddle.seed(args.seed)
+        random.seed(args.seed)
+        np.random.seed(args.seed)    
+        paddle.framework.random._manual_program_seed(args.seed)
+    

test_tipc/compare_results.py

@@ -0,0 +1,55 @@
+import numpy as np
+import os
+import subprocess
+import json
+import argparse
+import glob
+
+
+def init_args():
+    parser = argparse.ArgumentParser()
+    # params for testing assert allclose
+    parser.add_argument("--atol", type=float, default=1e-3)
+    parser.add_argument("--rtol", type=float, default=1e-3)
+    parser.add_argument("--gt_file", type=str, default="")
+    parser.add_argument("--log_file", type=str, default="")
+    parser.add_argument("--precision", type=str, default="fp32")
+    return parser
+
+def parse_args():
+    parser = init_args()
+    return parser.parse_args()
+
+def load_from_file(gt_file):
+    if not os.path.exists(gt_file):
+        raise ValueError("The log file {} does not exists!".format(gt_file))
+    with open(gt_file, 'r') as f:
+        data = f.readlines()
+        f.close()
+    parser_gt = {}
+    for line in data:
+        metric_name, result = line.strip("\n").split(":")
+        parser_gt[metric_name] = float(result)
+    return parser_gt
+
+if __name__ == "__main__":
+    # Usage:
+    # python3.7 test_tipc/compare_results.py --gt_file=./test_tipc/results/*.txt  --log_file=./test_tipc/output/*/*.txt
+
+    args = parse_args()
+
+    gt_collection = load_from_file(args.gt_file)
+    pre_collection = load_from_file(args.log_file)
+
+    for metric in pre_collection.keys():
+        try:
+            np.testing.assert_allclose(
+                np.array(pre_collection[metric]), np.array(gt_collection[metric]), atol=args.atol, rtol=args.rtol)
+            print(
+                "Assert allclose passed! The results of {} are consistent!".
+                format(metric))
+        except Exception as E:
+            print(E)
+            raise ValueError(
+                "The results of {} are inconsistent!".
+                format(metric))

test_tipc/configs/basicvsr/train_infer_python.txt

@@ -4,7 +4,7 @@ python:python3.7
 gpu_list:0
 ##
 auto_cast:null
-total_iters:lite_train_lite_infer=5|whole_train_whole_infer=200
+total_iters:lite_train_lite_infer=10|whole_train_whole_infer=200
 output_dir:./output/
 dataset.train.batch_size:lite_train_lite_infer=1|whole_train_whole_infer=1
 pretrained_model:null
@@ -13,7 +13,7 @@ train_infer_img_dir:./data/basicvsr_reds/test
 null:null
 ##
 trainer:norm_train
-norm_train:tools/main.py -c configs/basicvsr_reds.yaml -o dataset.train.dataset.num_clips=2
+norm_train:tools/main.py -c configs/basicvsr_reds.yaml --seed 123 -o dataset.train.dataset.num_clips=2 dataset.train.num_workers=0 log_config.interval=1 snapshot_config.interval=5 
 pact_train:null
 fpgm_train:null
 distill_train:null
@@ -37,7 +37,7 @@ inference_dir:basicvsrmodel_generator
 train_model:./inference/basicvsr/basicvsrmodel_generator
 infer_export:null
 infer_quant:False
-inference:tools/inference.py --model_type basicvsr -c configs/basicvsr_reds.yaml  -o dataset.test.num_clips=2 dataset.test.number_frames=6
+inference:tools/inference.py --model_type basicvsr -c configs/basicvsr_reds.yaml --seed 123 -o dataset.test.num_clips=2 dataset.test.number_frames=6 --output_path test_tipc/output/
 --device:gpu
 null:null
 null:null

test_tipc/configs/cyclegan/train_infer_python.txt

@@ -4,7 +4,7 @@ python:python3.7
 gpu_list:0|0,1
 ##
 auto_cast:null
-epochs:lite_train_lite_infer=5|whole_train_whole_infer=200
+epochs:lite_train_lite_infer=1|whole_train_whole_infer=200
 output_dir:./output/
 dataset.train.batch_size:lite_train_lite_infer=1|whole_train_whole_infer=1
 pretrained_model:null
@@ -13,7 +13,7 @@ train_infer_img_dir:./data/horse2zebra/test
 null:null
 ##
 trainer:norm_train
-norm_train:tools/main.py -c configs/cyclegan_horse2zebra.yaml -o
+norm_train:tools/main.py -c configs/cyclegan_horse2zebra.yaml --seed 123 -o log_config.interval=10 snapshot_config.interval=1
 pact_train:null
 fpgm_train:null
 distill_train:null
@@ -37,7 +37,7 @@ inference_dir:cycleganmodel_netG_A
 train_model:./inference/cyclegan_horse2zebra/cycleganmodel_netG_A
 infer_export:null
 infer_quant:False
-inference:tools/inference.py --model_type cyclegan -c configs/cyclegan_horse2zebra.yaml
+inference:tools/inference.py --model_type cyclegan --seed 123 -c configs/cyclegan_horse2zebra.yaml --output_path test_tipc/output/
 --device:gpu
 null:null
 null:null

test_tipc/configs/fom/train_infer_python.txt

@@ -4,7 +4,7 @@ python:python3.7
 gpu_list:0
 ##
 auto_cast:null
-epochs:lite_train_lite_infer=10|whole_train_whole_infer=100
+epochs:lite_train_lite_infer=1|whole_train_whole_infer=100
 output_dir:./output/
 dataset.train.batch_size:lite_train_lite_infer=8|whole_train_whole_infer=8
 pretrained_model:null
@@ -13,7 +13,7 @@ train_infer_img_dir:./data/firstorder_vox_256/test
 null:null
 ##
 trainer:norm_train
-norm_train:tools/main.py -c configs/firstorder_vox_256.yaml -o 
+norm_train:tools/main.py -c configs/firstorder_vox_256.yaml --seed 123 -o dataset.train.num_workers=0 log_config.interval=1 snapshot_config.interval=1 dataset.train.num_repeats=1 dataset.train.id_sampling=False
 pact_train:null
 fpgm_train:null
 distill_train:null
@@ -37,7 +37,7 @@ inference_dir:fom_dy2st
 train_model:./inference/fom_dy2st/
 infer_export:null
 infer_quant:False
-inference:tools/fom_infer.py --driving_path data/first_order/Voxceleb/test --output_path infer_output/fom
+inference:tools/fom_infer.py --driving_path data/first_order/Voxceleb/test --output_path test_tipc/output/fom/
 --device:gpu
 null:null
 null:null

test_tipc/configs/pix2pix/train_infer_python.txt

@@ -1,10 +1,10 @@
 ===========================train_params===========================
 model_name:pix2pix
 python:python3.7
-gpu_list:0|0,1
+gpu_list:0
 ##
 auto_cast:null
-epochs:lite_train_lite_infer=5|whole_train_whole_infer=200
+epochs:lite_train_lite_infer=10|whole_train_whole_infer=200
 output_dir:./output/
 dataset.train.batch_size:lite_train_lite_infer=1|whole_train_whole_infer=1
 pretrained_model:null
@@ -13,7 +13,7 @@ train_infer_img_dir:./data/facades/test
 null:null
 ##
 trainer:norm_train
-norm_train:tools/main.py -c configs/pix2pix_facades.yaml -o
+norm_train:tools/main.py -c configs/pix2pix_facades.yaml --seed 123 -o dataset.train.num_workers=0 log_config.interval=5
 pact_train:null
 fpgm_train:null
 distill_train:null
@@ -27,7 +27,7 @@ null:null
 ===========================infer_params===========================
 --output_dir:./output/
 load:null
-norm_export:tools/export_model.py -c configs/pix2pix_facades.yaml  --inputs_size="-1,3,-1,-1" --load 
+norm_export:tools/export_model.py -c configs/pix2pix_facades.yaml --inputs_size="-1,3,-1,-1" --load 
 quant_export:null 
 fpgm_export:null
 distill_export:null
@@ -37,7 +37,7 @@ inference_dir:pix2pixmodel_netG
 train_model:./inference/pix2pix_facade/pix2pixmodel_netG
 infer_export:null
 infer_quant:False
-inference:tools/inference.py --model_type pix2pix -c configs/pix2pix_facades.yaml
+inference:tools/inference.py --model_type pix2pix --seed 123 -c configs/pix2pix_facades.yaml --output_path test_tipc/output/
 --device:cpu
 null:null
 null:null

test_tipc/configs/stylegan2/train_infer_python.txt

@@ -4,7 +4,7 @@ python:python3.7
 gpu_list:0
 ##
 auto_cast:null
-total_iters::lite_train_lite_infer=10|whole_train_whole_infer=800
+total_iters:lite_train_lite_infer=10|whole_train_whole_infer=800
 output_dir:./output/
 dataset.train.batch_size:lite_train_lite_infer=3|whole_train_whole_infer=3
 pretrained_model:null
@@ -13,7 +13,7 @@ train_infer_img_dir:null
 null:null
 ##
 trainer:norm_train
-norm_train:tools/main.py -c configs/stylegan_v2_256_ffhq.yaml -o
+norm_train:tools/main.py -c configs/stylegan_v2_256_ffhq.yaml --seed 123 -o dataset.train.num_workers=0 log_config.interval=1 snapshot_config.interval=10
 pact_train:null
 fpgm_train:null
 distill_train:null
@@ -37,7 +37,7 @@ inference_dir:stylegan2model_gen
 train_model:./inference/stylegan2/stylegan2model_gen
 infer_export:null
 infer_quant:False
-inference:tools/inference.py --model_type stylegan2 -c configs/stylegan_v2_256_ffhq.yaml
+inference:tools/inference.py --model_type stylegan2 --seed 123 -c configs/stylegan_v2_256_ffhq.yaml --output_path test_tipc/output/
 --device:gpu
 null:null
 null:null