Skip to content

Commit aa74a80

Browse files
authored
【benchmark】fix gpu_mem unit (#427)
* add max_mem_reserved for benchmark * add max_mem_reserved for benchmark * fix
1 parent e323d98 commit aa74a80

File tree

1 file changed

+10
-8
lines changed

1 file changed

+10
-8
lines changed

paddle3d/apis/trainer.py

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -381,16 +381,18 @@ def train(self):
381381
tag='Training/learning_rate',
382382
value=lr,
383383
step=self.cur_iter)
384-
max_mem_reserved = paddle.device.cuda.max_memory_reserved()
385-
max_mem_allocated = paddle.device.cuda.max_memory_allocated(
386-
)
384+
max_mem_reserved_str = ""
385+
max_mem_allocated_str = ""
386+
if paddle.device.is_compiled_with_cuda():
387+
max_mem_reserved_str = f"max_mem_reserved: {paddle.device.cuda.max_memory_reserved() // (1024 ** 2)} MB,"
388+
max_mem_allocated_str = f"max_mem_allocated: {paddle.device.cuda.max_memory_allocated() // (1024 ** 2)} MB"
387389
self.logger.info(
388390
'[TRAIN] epoch={}/{}, iter={}/{} {}, lr={:.6f}, batch_cost: {:.6f} sec, '
389-
'ips: {:.6f} images/s | ETA {}, max_mem_reserved: {} B, max_mem_allocated: {} B'
390-
.format(self.cur_epoch, self.epochs, self.cur_iter,
391-
self.iters, loss_log, lr, timer.speed,
392-
timer.ips, timer.eta, max_mem_reserved,
393-
max_mem_allocated))
391+
'ips: {:.6f} images/s | ETA {}, {} {}'.format(
392+
self.cur_epoch, self.epochs, self.cur_iter,
393+
self.iters, loss_log, lr, timer.speed, timer.ips,
394+
timer.eta, max_mem_reserved_str,
395+
max_mem_allocated_str))
394396

395397
losses_sum.clear()
396398

0 commit comments

Comments
 (0)