Tensorboard validation log fix

- Fix validation logs which were again improperly logged in Tensorboard with Tensorflow
- Increase frequency of mid-validation checks during testing

Author: jamesdolezal

slideflow/model/tensorflow.py

@@ -670,11 +670,12 @@ def __init__(self, parent: "Trainer", cb_args: SimpleNamespace) -> None:
     def _log_training_metrics(self, logs):
         """Log training metrics to Tensorboard/Neptune."""
         # Log to Tensorboard.
-        for _log in logs:
-            tf.summary.scalar(
-                f'batch_{_log}',
-                data=logs[_log],
-                step=self.global_step)
+        with self.train_summary_writer.as_default():
+            for _log in logs:
+                tf.summary.scalar(
+                    f'batch_{_log}',
+                    data=logs[_log],
+                    step=self.global_step)
         # Log to neptune.
         if self.neptune_run:
             self.neptune_run['metrics/train/batch/loss'].log(
@@ -890,10 +891,13 @@ def on_train_batch_end(self, batch: int, logs={}) -> None:
                 verbosity='quiet',
             )
             val_metrics = {'loss': loss}
+            val_log_metrics = {'loss': loss}
             if isinstance(acc, float):
                 val_metrics['accuracy'] = acc
+                val_log_metrics['accuracy'] = acc
             elif acc is not None:
                 val_metrics.update({f'accuracy-{i+1}': acc[i] for i in range(len(acc))})
+                val_log_metrics.update({f'out-{i}_accuracy': acc[i] for i in range(len(acc))})
 
             val_loss = val_metrics['loss']
             self.model.stop_training = False
@@ -920,7 +924,7 @@ def on_train_batch_end(self, batch: int, logs={}) -> None:
                 print('\r\033[K', end='')
             self.moving_average += [early_stop_value]
 
-            self._log_validation_metrics(logs)
+            self._log_validation_metrics(val_log_metrics)
             # Log training metrics if not already logged this batch
             if batch % self.cb_args.log_frequency > 0:
                 self._log_training_metrics(logs)
@@ -1356,7 +1360,7 @@ def _verify_img_format(self, dataset: "sf.Dataset") -> None:
     def load(self, model: str) -> tf.keras.Model:
         self.model = load(
             model,
-            method=self.load_method, 
+            method=self.load_method,
             custom_objects=self.custom_objects
         )
 
@@ -2503,7 +2507,7 @@ def _predict(self, inp):
 
 
 def load(
-    path: str, 
+    path: str,
     method: str = 'full',
     custom_objects: Optional[Dict[str, Any]] = None,):
     """Load Tensorflow model from location.

slideflow/test/__init__.py

@@ -107,6 +107,13 @@ def __init__(
         # Rebuild tfrecord indices
         self.project.dataset(self.tile_px, 1208).build_index(True)
 
+        # Set up training keyword arguments.
+        self.train_kwargs = dict(
+            validate_on_batch=5,
+            steps_per_epoch_override=50,
+            save_predictions=True
+        )
+
     def _get_model(self, name: str, epoch: int = 1) -> str:
         assert self.project is not None
         prev_run_dirs = [
@@ -326,9 +333,6 @@ def train_perf(self, **train_kwargs) -> None:
                     exp_label='manual_hp',
                     outcomes='category1',
                     val_k=1,
-                    validate_on_batch=10,
-                    save_predictions=True,
-                    steps_per_epoch_override=20,
                     params='sweep.json',
                     pretrain=None,
                     **train_kwargs
@@ -374,6 +378,9 @@ def test_training(
                 additional slide-level input. Defaults to True.
         """
         assert self.project is not None
+        for k in self.train_kwargs:
+            if k not in train_kwargs:
+                train_kwargs[k] = self.train_kwargs[k]
         # Disable checkpoints for tensorflow backend, to save disk space
         if (sf.backend() == 'tensorflow'
            and 'save_checkpoints' not in train_kwargs):
@@ -408,9 +415,6 @@ def test_training(
                             outcomes='category1',
                             val_k=1,
                             params=hp,
-                            validate_on_batch=10,
-                            steps_per_epoch_override=20,
-                            save_predictions=True,
                             pretrain=None,
                             **resume_kw,
                             **train_kwargs
@@ -436,9 +440,6 @@ def test_training(
                         outcomes='category1',
                         val_k=1,
                         params=hp,
-                        validate_on_batch=10,
-                        steps_per_epoch_override=20,
-                        save_predictions=True,
                         pretrain=to_resume,
                         **train_kwargs
                     )
@@ -455,9 +456,6 @@ def test_training(
                         outcomes=['category1', 'category2'],
                         val_k=1,
                         params=self.setup_hp('categorical'),
-                        validate_on_batch=10,
-                        steps_per_epoch_override=20,
-                        save_predictions=True,
                         pretrain=None,
                         **train_kwargs
                     )
@@ -474,9 +472,6 @@ def test_training(
                         outcomes=['linear1'],
                         val_k=1,
                         params=self.setup_hp('linear'),
-                        validate_on_batch=10,
-                        steps_per_epoch_override=20,
-                        save_predictions=True,
                         pretrain=None,
                         **train_kwargs
                     )
@@ -493,9 +488,6 @@ def test_training(
                         outcomes=['linear1', 'linear2'],
                         val_k=1,
                         params=self.setup_hp('linear'),
-                        validate_on_batch=10,
-                        steps_per_epoch_override=20,
-                        save_predictions=True,
                         pretrain=None,
                         **train_kwargs
                     )
@@ -514,9 +506,6 @@ def test_training(
                         input_header='category2',
                         params=self.setup_hp('categorical'),
                         val_k=1,
-                        validate_on_batch=10,
-                        steps_per_epoch_override=20,
-                        save_predictions=True,
                         pretrain=None,
                         **train_kwargs
                     )
@@ -535,9 +524,6 @@ def test_training(
                             input_header='event',
                             params=self.setup_hp('cph'),
                             val_k=1,
-                            validate_on_batch=10,
-                            steps_per_epoch_override=20,
-                            save_predictions=True,
                             pretrain=None,
                             **train_kwargs
                         )
@@ -558,9 +544,6 @@ def test_training(
                             input_header=['event', 'category1'],
                             params=self.setup_hp('cph'),
                             val_k=1,
-                            validate_on_batch=10,
-                            steps_per_epoch_override=20,
-                            save_predictions=True,
                             pretrain=None,
                             **train_kwargs
                         )
@@ -581,9 +564,6 @@ def test_training(
                         outcomes='category1',
                         val_k=1,
                         params=hp,
-                        validate_on_batch=10,
-                        steps_per_epoch_override=20,
-                        save_predictions=True,
                         from_wsi=True,
                         pretrain=None,
                         **train_kwargs