[fix]: dataloader of list dataset has length.

Author: Liwb5

pgl/utils/data/dataloader.py

@@ -15,6 +15,7 @@
 """dataloader
 """
 import warnings
+import time
 import numpy as np
 from collections import namedtuple
 
@@ -104,15 +105,8 @@ def __init__(self,
             raise ValueError("num_workers(default: 1) should be larger than 0, " \
                         "but got [num_workers=%s] < 1." % self.num_workers)
 
-    def __len__(self):
-        if not isinstance(self.dataset, StreamDataset):
-            return len(self.sampler)
-        else:
-            raise "StreamDataset has no length"
-
-    def __iter__(self):
-        # generating a iterable sequence for produce batch data without repetition
         if isinstance(self.dataset, StreamDataset):  # for stream data
+            # generating a iterable sequence for produce batch data without repetition
             self.sampler = StreamSampler(
                 self.dataset,
                 batch_size=self.batch_size,
@@ -124,6 +118,16 @@ def __iter__(self):
                 drop_last=self.drop_last,
                 shuffle=self.shuffle)
 
+    def __len__(self):
+        if not isinstance(self.dataset, StreamDataset):
+            return len(self.sampler)
+        else:
+            raise "StreamDataset has no length"
+
+    def __iter__(self):
+        # random seed will be fixed when using multiprocess, 
+        # so set seed explicitly every time
+        np.random.seed()
         if self.num_workers == 1:
             r = paddle.reader.buffered(_DataLoaderIter(self, 0), self.buf_size)
         else:

pgl/utils/data/sampler.py

@@ -26,18 +26,13 @@ def __init__(self, dataset, batch_size=1, drop_last=False, shuffle=False):
         self.drop_last = drop_last
         self.shuffle = shuffle
 
-        length = len(self.dataset)
-        self.perm = np.arange(0, length)
-
-        # shuffle one time whne Sampler is created
+    def __iter__(self):
+        perm = np.arange(0, len(self.dataset))
         if self.shuffle:
-            seed = int(float(time.time()) * 1000) % 10000007
-            np.random.seed(seed)
-            np.random.shuffle(self.perm)
+            np.random.shuffle(perm)
 
-    def __iter__(self):
         batch = []
-        for idx in self.perm:
+        for idx in perm:
             batch.append(idx)
             if len(batch) == self.batch_size:
                 yield batch