New SimulationTimeSeriesOneByOne plugin (#1217)

* Initial commit

* Improvements to the data provider structure, black and isort

* Fixed vector selector problem and added visualization settings group

* Implemented the rest of the settings groups

* New folder _view_elements

* New view element for the bottom visualization

* Further work on converting the callbacks to WLF

* First working version of callback, plenty of bugs still

* Various improvements, made the time series figure local to the plugin to avoid problems with other plugins

* pylint and mypy fixes

* Fixed scale bug

* Various improvements

* Changelog entry

* Fixed pylint issue

* fixed changelog issue

* Added test

* Deprecated ReservoirSimulationTimeseriesOnyByOne plugin

* Review related updates

* Fixed link in changelog

---------

Co-authored-by: Øyvind Lind-Johansen <olind@equinor.com>

Author: lindjoha

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED] - YYYY-MM-DD
 
+### Added
+- [#1217](https://github.com/equinor/webviz-subsurface/pull/1217) - New plugin `SimulationTimeSeriesOneByOne`, meant to replace the old `ReservoirSimulationTimeSeriesOneByOne`. Uses the `.arrow` summary provider and is implemented with WLF (Webviz Layout Framework).
+
+
 ## [0.2.19] - 2023-05-05
 
 ### Changed

setup.py

@@ -67,6 +67,7 @@
             "SegyViewer = webviz_subsurface.plugins._segy_viewer:SegyViewer",
             "SeismicMisfit = webviz_subsurface.plugins._seismic_misfit:SeismicMisfit",
             "SimulationTimeSeries = webviz_subsurface.plugins._simulation_time_series:SimulationTimeSeries",
+            "SimulationTimeSeriesOneByOne = webviz_subsurface.plugins._simulation_time_series_onebyone:SimulationTimeSeriesOneByOne",
             "StructuralUncertainty = webviz_subsurface.plugins._structural_uncertainty:StructuralUncertainty",
             "SubsurfaceMap = webviz_subsurface.plugins._subsurface_map:SubsurfaceMap",
             "SurfaceViewerFMU = webviz_subsurface.plugins._surface_viewer_fmu:SurfaceViewerFMU",

tests/integration_tests/plugin_tests/test_simulation_timeseries_onebyone.py

@@ -0,0 +1,23 @@
+import warnings
+
+# pylint: disable=no-name-in-module
+from webviz_config.plugins import SimulationTimeSeriesOneByOne
+from webviz_config.testing import WebvizComposite
+
+
+def test_simulation_timeseries_onebyone(
+    _webviz_duo: WebvizComposite, shared_settings: dict
+) -> None:
+    plugin = SimulationTimeSeriesOneByOne(
+        webviz_settings=shared_settings["SENS_SETTINGS"],
+        ensembles=shared_settings["SENS_ENSEMBLES"],
+        initial_vector="FOPT",
+    )
+    _webviz_duo.start_server(plugin)
+    logs = []
+    for log in _webviz_duo.get_logs():
+        if "dash_renderer" in log.get("message"):
+            warnings.warn(log.get("message"))
+        else:
+            logs.append(log)
+    assert not logs

webviz_subsurface/_components/tornado/_tornado_table.py

@@ -23,6 +23,7 @@ def __init__(
             lambda x: str(len(x))
         )
         self._table["Response"] = tornado_data.response_name
+        self._table["Reference"] = tornado_data.reference_average
         self._table.rename(
             columns={
                 "sensname": "Sensitivity",
@@ -64,6 +65,7 @@ def columns(self) -> List[Dict]:
                 "True high",
                 "Low #reals",
                 "High #reals",
+                "Reference",
             ]
         ]
 

webviz_subsurface/plugins/_reservoir_simulation_timeseries_onebyone.py

@@ -11,6 +11,7 @@
 from dash.exceptions import PreventUpdate
 from webviz_config import WebvizPluginABC, WebvizSettings
 from webviz_config.common_cache import CACHE
+from webviz_config.deprecation_decorators import deprecated_plugin
 from webviz_config.webviz_store import webvizstore
 
 from webviz_subsurface._components import TornadoWidget
@@ -33,6 +34,9 @@
 
 
 # pylint: disable=too-many-instance-attributes
+@deprecated_plugin(
+    "This plugin has been replaced by the plugin `SimulationTimeSeriesOneByOne`"
+)
 class ReservoirSimulationTimeSeriesOneByOne(WebvizPluginABC):
     """Visualizes reservoir simulation time series data for sensitivity studies based \
 on a design matrix.

webviz_subsurface/plugins/_simulation_time_series_onebyone/__init__.py

@@ -0,0 +1 @@
+from ._plugin import SimulationTimeSeriesOneByOne

webviz_subsurface/plugins/_simulation_time_series_onebyone/_plugin.py

@@ -0,0 +1,154 @@
+from pathlib import Path
+from typing import Dict
+
+import pandas as pd
+from webviz_config import WebvizPluginABC, WebvizSettings
+from webviz_config.utils import StrEnum
+
+from webviz_subsurface._models.parameter_model import ParametersModel
+from webviz_subsurface._providers import (
+    EnsembleTableProvider,
+    EnsembleTableProviderFactory,
+    Frequency,
+)
+from webviz_subsurface._utils.ensemble_summary_provider_set_factory import (
+    create_lazy_ensemble_summary_provider_set_from_paths,
+    create_presampled_ensemble_summary_provider_set_from_paths,
+)
+
+from ._utils import SimulationTimeSeriesOneByOneDataModel
+from ._views._onebyone_view import OneByOneView
+
+
+class SimulationTimeSeriesOneByOne(WebvizPluginABC):
+    """Visualizes reservoir simulation time series data for sensitivity studies based \
+on a design matrix.
+A tornado plot can be calculated interactively for each date/vector by selecting a date.
+After selecting a date individual sensitivities can be selected to highlight the realizations
+run with that sensitivity.
+---
+**Input arguments**
+* **`ensembles`:** Which ensembles in `shared_settings` to visualize.
+* **`rel_file_pattern`:** Path to `.arrow` files with summary data.
+* **`sampling`:** Time separation between extracted values. Can be e.g. `monthly` (default) or \
+    `yearly`.
+* **`perform_presampling`:** Presample summary data instead of lazy sampling.
+* **`initial_vector`:** Initial vector to display
+* **`line_shape_fallback`:** Fallback interpolation method between points. Vectors identified as \
+    rates or phase ratios are always backfilled, vectors identified as cumulative (totals) are \
+    always linearly interpolated. The rest use the fallback.
+    Supported options:
+    * `linear` (default)
+    * `backfilled`
+    * `hv`, `vh`, `hvh`, `vhv` and `spline` (regular Plotly options).
+"""
+
+    class Ids(StrEnum):
+        ONEBYONE_VIEW = "onebyone-view"
+
+    def __init__(
+        self,
+        webviz_settings: WebvizSettings,
+        ensembles: list,
+        rel_file_pattern: str = "share/results/unsmry/*.arrow",
+        sampling: str = Frequency.MONTHLY.value,
+        perform_presampling: bool = False,
+        initial_vector: str = None,
+        line_shape_fallback: str = "linear",
+    ) -> None:
+        super().__init__()
+
+        # vectormodel: ProviderTimeSeriesDataModel
+        table_provider = EnsembleTableProviderFactory.instance()
+        resampling_frequency = Frequency(sampling)
+
+        if ensembles is not None:
+            ensemble_paths: Dict[str, Path] = {
+                ensemble_name: webviz_settings.shared_settings["scratch_ensembles"][
+                    ensemble_name
+                ]
+                for ensemble_name in ensembles
+            }
+            if perform_presampling:
+                self._presampled_frequency = resampling_frequency
+                summary_provider_set = (
+                    create_presampled_ensemble_summary_provider_set_from_paths(
+                        ensemble_paths, rel_file_pattern, self._presampled_frequency
+                    )
+                )
+            else:
+                summary_provider_set = (
+                    create_lazy_ensemble_summary_provider_set_from_paths(
+                        ensemble_paths, rel_file_pattern
+                    )
+                )
+        else:
+            raise ValueError('Incorrect argument, must provide "ensembles"')
+
+        if not summary_provider_set:
+            raise ValueError(
+                "Initial provider set is undefined, and ensemble summary providers"
+                " are not instantiated for plugin"
+            )
+
+        parameterproviderset = {
+            ens_name: table_provider.create_from_per_realization_parameter_file(
+                str(ens_path)
+            )
+            for ens_name, ens_path in ensemble_paths.items()
+        }
+        parameter_df = create_df_from_table_provider(parameterproviderset)
+        parametermodel = ParametersModel(dataframe=parameter_df, drop_constants=True)
+
+        self.add_view(
+            OneByOneView(
+                data_model=SimulationTimeSeriesOneByOneDataModel(
+                    provider_set=summary_provider_set,
+                    parametermodel=parametermodel,
+                    webviz_settings=webviz_settings,
+                    resampling_frequency=resampling_frequency,
+                    initial_vector=initial_vector,
+                    line_shape_fallback=line_shape_fallback,
+                ),
+            ),
+            self.Ids.ONEBYONE_VIEW,
+        )
+
+    # @property
+    # def tour_steps(self) -> List[dict]:
+    #     return [
+    #         {
+    #             "id": self.uuid("layout"),
+    #             "content": (
+    #                 "Dashboard displaying time series from a sensitivity study."
+    #             ),
+    #         },
+    #         {
+    #             "id": self.uuid("graph-wrapper"),
+    #             "content": (
+    #                 "Selected time series displayed per realization. "
+    #                 "Click in the plot to calculate tornadoplot for the "
+    #                 "corresponding date, then click on the tornado plot to "
+    #                 "highlight the corresponding sensitivity."
+    #             ),
+    #         },
+    #         {
+    #             "id": self.uuid("table"),
+    #             "content": (
+    #                 "Table statistics for all sensitivities for the selected date."
+    #             ),
+    #         },
+    #         {"id": self.uuid("vector"), "content": "Select time series"},
+    #         {"id": self.uuid("ensemble"), "content": "Select ensemble"},
+    #     ]
+
+
+def create_df_from_table_provider(
+    providerset: Dict[str, EnsembleTableProvider]
+) -> pd.DataFrame:
+    dfs = []
+    for ens, provider in providerset.items():
+        df = provider.get_column_data(column_names=provider.column_names())
+        df["ENSEMBLE"] = ens
+        dfs.append(df)
+    return pd.concat(dfs)

webviz_subsurface/plugins/_simulation_time_series_onebyone/_types.py

@@ -0,0 +1,18 @@
+from webviz_config.utils import StrEnum
+
+
+class LineType(StrEnum):
+    REALIZATION = "realizations"
+    STATISTICS = "statistics"
+
+
+class ScaleType(StrEnum):
+    PERCENTAGE = "Percentage"
+    ABSOLUTE = "Absolute"
+    TRUE_VALUE = "True"
+
+
+class LabelOptions(StrEnum):
+    DETAILED = "detailed"
+    SIMPLE = "simple"
+    HIDE = "hide"

webviz_subsurface/plugins/_simulation_time_series_onebyone/_utils/__init__.py

@@ -0,0 +1,7 @@
+from ._datetime_utils import date_from_str, date_to_str
+from ._simulation_time_series_onebyone_datamodel import (
+    SimulationTimeSeriesOneByOneDataModel,
+    create_tornado_table,
+    create_vector_selector_data,
+    get_tornado_data,
+)

webviz_subsurface/plugins/_simulation_time_series_onebyone/_utils/_datetime_utils.py

@@ -0,0 +1,14 @@
+import datetime
+
+
+def date_from_str(date_str: str) -> datetime.datetime:
+    return datetime.datetime.strptime(date_str, "%Y-%m-%d")
+
+
+def date_to_str(date: datetime.datetime) -> str:
+    if date.hour != 0 or date.minute != 0 or date.second != 0 or date.microsecond != 0:
+        raise ValueError(
+            f"Invalid date resolution, expected no data for hour, minute, second"
+            f" or microsecond for {str(date)}"
+        )
+    return date.strftime("%Y-%m-%d")