integrations: Standardize metric name to group under subfolders.

Author: daavoo

dvclive/fastai.py

@@ -1,6 +1,7 @@
 from fastai.callback.core import Callback
 
 from dvclive import Live
+from dvclive.utils import standardize_metric_name
 
 
 class DvcLiveCallback(Callback):
@@ -13,8 +14,9 @@ def after_epoch(self):
         for key, value in zip(
             self.learn.recorder.metric_names, self.learn.recorder.log
         ):
-            key = key.replace("_", "/")
-            self.dvclive.log(f"{key}", float(value))
+            self.dvclive.log(
+                standardize_metric_name(key, __name__), float(value)
+            )
 
         if self.model_file:
             self.learn.save(self.model_file)

dvclive/huggingface.py

@@ -6,6 +6,7 @@
 )
 
 from dvclive import Live
+from dvclive.utils import standardize_metric_name
 
 
 class DvcLiveCallback(TrainerCallback):
@@ -23,7 +24,7 @@ def on_log(
     ):
         logs = kwargs["logs"]
         for key, value in logs.items():
-            self.dvclive.log(key, value)
+            self.dvclive.log(standardize_metric_name(key, __name__), value)
         self.dvclive.next_step()
 
     def on_epoch_end(

dvclive/keras.py

@@ -8,6 +8,7 @@
 )
 
 from dvclive import Live
+from dvclive.utils import standardize_metric_name
 
 
 class DvcLiveCallback(Callback):
@@ -39,7 +40,7 @@ def on_epoch_end(
     ):  # pylint: disable=unused-argument
         logs = logs or {}
         for metric, value in logs.items():
-            self.dvclive.log(metric, value)
+            self.dvclive.log(standardize_metric_name(metric, __name__), value)
         if self.model_file:
             if self.save_weights_only:
                 self.model.save_weights(self.model_file)

dvclive/lightning.py

@@ -3,16 +3,13 @@
 from pytorch_lightning.loggers import LightningLoggerBase
 from pytorch_lightning.loggers.base import rank_zero_experiment
 from pytorch_lightning.utilities import rank_zero_only
-from pytorch_lightning.utilities.logger import _add_prefix
 from torch import is_tensor
 
 from dvclive import Live
+from dvclive.utils import standardize_metric_name
 
 
 class DvcLiveLogger(LightningLoggerBase):
-
-    LOGGER_JOIN_CHAR = "-"
-
     def __init__(
         self,
         run_name: Optional[str] = "dvclive_run",
@@ -70,9 +67,9 @@ def log_metrics(
             rank_zero_only.rank == 0
         ), "experiment tried to log from global_rank != 0"
 
-        metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR)
         for metric_name, metric_val in metrics.items():
             if is_tensor(metric_val):
                 metric_val = metric_val.cpu().detach().item()
+            metric_name = standardize_metric_name(metric_name, __name__)
             self.experiment.log(name=metric_name, val=metric_val)
         self.experiment.next_step()

dvclive/utils.py

@@ -73,3 +73,32 @@ def env2bool(var, undefined=False):
     if var is None:
         return undefined
     return bool(re.search("1|y|yes|true", var, flags=re.I))
+
+
+def standardize_metric_name(metric_name: str, framework: str) -> str:
+    """Map framework-specific format to DVCLive standard.
+
+    Use `{split}/` as prefix in order to seperate by subfolders.
+    Use `{train|eval}` as split name.
+    """
+    if framework == "dvclive.fastai":
+        metric_name = metric_name.replace("train_", "train/")
+        metric_name = metric_name.replace("valid_", "eval/")
+
+    elif framework == "dvclive.huggingface":
+        for split in {"train", "eval"}:
+            metric_name = metric_name.replace(f"{split}_", f"{split}/")
+
+    elif framework == "dvclive.keras":
+        if "val_" in metric_name:
+            metric_name = metric_name.replace("val_", "eval/")
+        else:
+            metric_name = f"train/{metric_name}"
+
+    elif framework == "dvclive.lightning":
+        parts = metric_name.split("_")
+        if len(parts) > 2:
+            split, *rest, freq = parts
+            metric_name = f"{split}/{freq}/{'_'.join(rest)}"
+
+    return metric_name

tests/test_fastai.py

@@ -48,7 +48,7 @@ def test_fastai_callback(tmp_dir, data_loader):
     assert os.path.exists("dvclive")
 
     train_path = tmp_dir / "dvclive" / Scalar.subfolder / "train"
-    valid_path = tmp_dir / "dvclive" / Scalar.subfolder / "valid"
+    valid_path = tmp_dir / "dvclive" / Scalar.subfolder / "eval"
 
     assert train_path.is_dir()
     assert valid_path.is_dir()

tests/test_huggingface.py

@@ -82,10 +82,11 @@ def test_huggingface_integration(tmp_dir, model, args, data, tokenizer):
     logs, _ = read_logs(tmp_dir / "dvclive" / Scalar.subfolder)
 
     assert len(logs) == 10
-    assert "eval_matthews_correlation" in logs
-    assert "eval_loss" in logs
+    assert os.path.join("eval", "matthews_correlation") in logs
+    assert os.path.join("eval", "loss") in logs
+    assert os.path.join("train", "loss") in logs
     assert len(logs["epoch"]) == 3
-    assert len(logs["eval_loss"]) == 2
+    assert len(logs[os.path.join("eval", "loss")]) == 2
 
 
 def test_huggingface_model_file(tmp_dir, model, args, data, tokenizer, mocker):

tests/test_keras.py

@@ -42,13 +42,15 @@ def test_keras_callback(tmp_dir, xor_model, capture_wrap):
         y,
         epochs=1,
         batch_size=1,
+        validation_split=0.2,
         callbacks=[DvcLiveCallback()],
     )
 
     assert os.path.exists("dvclive")
     logs, _ = read_logs(tmp_dir / "dvclive" / Scalar.subfolder)
 
-    assert "accuracy" in logs
+    assert os.path.join("train", "accuracy") in logs
+    assert os.path.join("eval", "accuracy") in logs
 
 
 @pytest.mark.parametrize("save_weights_only", (True, False))

tests/test_lightning.py

@@ -96,6 +96,6 @@ def test_lightning_integration(tmp_dir):
     logs, _ = read_logs(tmp_dir / "logs" / Scalar.subfolder)
 
     assert len(logs) == 3
-    assert "train_loss_step" in logs
-    assert "train_loss_epoch" in logs
+    assert os.path.join("train", "epoch", "loss") in logs
+    assert os.path.join("train", "step", "loss") in logs
     assert "epoch" in logs