diff --git a/src/everest/api/everest_data_api.py b/src/everest/api/everest_data_api.py index 7339af52762..d94d0766ac2 100644 --- a/src/everest/api/everest_data_api.py +++ b/src/everest/api/everest_data_api.py @@ -1,9 +1,9 @@ from collections import OrderedDict import pandas as pd +from ert.storage import open_storage from seba_sqlite.snapshot import SebaSnapshot -from ert.storage import open_storage from everest.config import EverestConfig from everest.detached import ServerStatus, everserver_status @@ -158,9 +158,10 @@ def summary_values(self, batches=None, keys=None): data_frames = [] storage = open_storage(self._config.storage_dir, "r") for batch_id in batches: - summary = storage.get_ensemble_by_name( - f"batch_{batch_id}" - ).load_all_summary_data() + case_name = f"batch_{batch_id}" + experiment = storage.get_experiment_by_name(f"experiment_{case_name}") + ensemble = experiment.get_ensemble_by_name(case_name) + summary = ensemble.load_all_summary_data() if not summary.empty: columns = set(summary.columns) if keys is not None: diff --git a/src/everest/bin/everest_script.py b/src/everest/bin/everest_script.py index cf5c04e086d..850fec02ceb 100755 --- a/src/everest/bin/everest_script.py +++ b/src/everest/bin/everest_script.py @@ -9,6 +9,7 @@ from ert.config import ErtConfig from ert.storage import open_storage + from everest.config import EverestConfig from everest.detached import ( ServerStatus, diff --git a/src/everest/bin/everload_script.py b/src/everest/bin/everload_script.py index cd83799ecc6..a03994691a9 100755 --- a/src/everest/bin/everload_script.py +++ b/src/everest/bin/everload_script.py @@ -10,6 +10,7 @@ from ert import LibresFacade from ert.config import ErtConfig from ert.storage import open_storage + from everest import MetaDataColumnNames as MDCN from everest import export from everest.config import EverestConfig @@ -59,10 +60,11 @@ def everload_entry(args=None): backup_path = None if not os.path.isdir(storage_path): storage_path = None - elif not options.overwrite: - backup_path = storage_path + datetime.datetime.utcnow().strftime( - "__%Y-%m-%d_%H.%M.%S.%f" - ) + else: # storage_path exists, we may want to back it up + if not options.overwrite: + backup_path = storage_path + datetime.datetime.utcnow().strftime( + "__%Y-%m-%d_%H.%M.%S.%f" + ) if not options.silent and not user_confirms(sim_dir, storage_path, backup_path): return @@ -178,17 +180,18 @@ def reload_data(ever_config: EverestConfig, backup_path=None): def _internalize_batch(ert_config, batch_id, batch_data): facade = LibresFacade(ert_config) - batch_name = "batch_{}".format(batch_id) + case_name = "batch_{}".format(batch_id) batch_size = batch_data.shape[0] with open_storage(facade.enspath, "w") as storage: - file_system = storage.get_ensemble_by_name(batch_name) + experiment = storage.get_experiment_by_name(f"experiment_{case_name}") + ensemble = experiment.get_ensemble_by_name(case_name) # Everest artificially inflates the ensemble size as it is not possible to # add after the fact, therefore a batch is much smaller than the overall # ensemble size realizations = [True] * batch_size + [False] * ( facade.get_ensemble_size() - batch_size ) - facade.load_from_forward_model(file_system, realizations, 0) + facade.load_from_forward_model(ensemble, realizations, 0) if __name__ == "__main__": diff --git a/src/everest/bin/main.py b/src/everest/bin/main.py index 334d0092cdd..ca8b64440c0 100644 --- a/src/everest/bin/main.py +++ b/src/everest/bin/main.py @@ -3,6 +3,8 @@ import logging import sys +from ieverest.bin.ieverest_script import ieverest_entry + from everest import __version__ as everest_version from everest import docs from everest.bin.config_branch_script import config_branch_entry @@ -15,7 +17,6 @@ from everest.bin.monitor_script import monitor_entry from everest.bin.visualization_script import visualization_entry from everest.util import configure_logger -from ieverest.bin.ieverest_script import ieverest_entry def _create_dump_action(dumps, extended=False): diff --git a/src/everest/bin/utils.py b/src/everest/bin/utils.py index 78b6ce480b8..9d92ff524c2 100644 --- a/src/everest/bin/utils.py +++ b/src/everest/bin/utils.py @@ -8,8 +8,8 @@ import colorama from colorama import Fore - from ert.simulator.batch_simulator_context import Status + from everest.config import EverestConfig from everest.detached import ( OPT_PROGRESS_ID, diff --git a/src/everest/config/everest_config.py b/src/everest/config/everest_config.py index 1358a4cd4a6..f0ef3f5c33a 100644 --- a/src/everest/config/everest_config.py +++ b/src/everest/config/everest_config.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING, List, Literal, Optional, Protocol, no_type_check +from ert.config import ErtConfig from pydantic import ( AfterValidator, BaseModel, @@ -19,7 +20,6 @@ from ruamel.yaml import YAML, YAMLError from typing_extensions import Annotated -from ert.config import ErtConfig from everest.config.control_variable_config import ControlVariableGuessListConfig from everest.config.install_template_config import InstallTemplateConfig from everest.config.server_config import ServerConfig diff --git a/src/everest/config/input_constraint_config.py b/src/everest/config/input_constraint_config.py index ab7b32d84d7..3a5f82809c5 100644 --- a/src/everest/config/input_constraint_config.py +++ b/src/everest/config/input_constraint_config.py @@ -9,9 +9,9 @@ class InputConstraintConfig(BaseModel, extra="forbid"): # type: ignore If we are trying to constrain only one control (i.e the z control) value: | input_constraints: | - weights: -| point_3D_0.x-0: 0 -| point_3D_0.y-1: 0 -| point_3D_0.z-2: 1 +| point_3D.x-0: 0 +| point_3D.y-1: 0 +| point_3D.z-2: 1 | upper_bound: 0.2 Only control values (x, y, z) that satisfy the following equation will be allowed: @@ -23,9 +23,9 @@ class InputConstraintConfig(BaseModel, extra="forbid"): # type: ignore description="""**Example** | input_constraints: | - weights: -| point_3D_0.x-0: 1 -| point_3D_0.y-1: 2 -| point_3D_0.z-2: 3 +| point_3D.x-0: 1 +| point_3D.y-1: 2 +| point_3D.z-2: 3 | target: 4 Only control values (x, y, z) that satisfy the following equation will be allowed: @@ -37,9 +37,9 @@ class InputConstraintConfig(BaseModel, extra="forbid"): # type: ignore description="""**Example** | input_constraints: | - weights: -| point_3D_0.x-0: 1 -| point_3D_0.y-1: 2 -| point_3D_0.z-2: 3 +| point_3D.x-0: 1 +| point_3D.y-1: 2 +| point_3D.z-2: 3 | lower_bound: 4 Only control values (x, y, z) that satisfy the following @@ -52,9 +52,9 @@ class InputConstraintConfig(BaseModel, extra="forbid"): # type: ignore description="""**Example** | input_constraints: | - weights: -| point_3D_0.x-0: 1 -| point_3D_0.y-1: 2 -| point_3D_0.z-2: 3 +| point_3D.x-0: 1 +| point_3D.y-1: 2 +| point_3D.z-2: 3 | upper_bound: 4 Only control values (x, y, z) that satisfy the following equation will be allowed: diff --git a/src/everest/detached/__init__.py b/src/everest/detached/__init__.py index f86d19ecaaf..c90e6b90afd 100644 --- a/src/everest/detached/__init__.py +++ b/src/everest/detached/__init__.py @@ -10,11 +10,11 @@ import pkg_resources import requests +from ert import BatchContext, BatchSimulator +from ert.config import ErtConfig, QueueSystem from seba_sqlite.exceptions import ObjectNotFoundError from seba_sqlite.snapshot import SebaSnapshot -from ert import BatchContext, BatchSimulator -from ert.config import ErtConfig, QueueSystem from everest.config import EverestConfig from everest.config_keys import ConfigKeys as CK from everest.simulator import JOB_FAILURE, JOB_SUCCESS, Status diff --git a/src/everest/docs/__main__.py b/src/everest/docs/__main__.py index 7288cef5177..f39ee1a0760 100644 --- a/src/everest/docs/__main__.py +++ b/src/everest/docs/__main__.py @@ -5,7 +5,7 @@ from everest.docs.generate_docs_from_config_spec import generate_docs_pydantic_to_rst committed_file = os.path.abspath( - relpath("..", "..", "docs", "source", "config_generated.rst") + relpath("..", "docs", "source", "config_generated.rst") ) print(f"Writing new docs contents to {committed_file}") diff --git a/src/everest/export.py b/src/everest/export.py index 97a195ae7f8..0056a8e76c0 100644 --- a/src/everest/export.py +++ b/src/everest/export.py @@ -4,10 +4,10 @@ from typing import Any, Dict, List, Optional, Set import pandas as pd +from ert.storage import open_storage from pandas import DataFrame from seba_sqlite.snapshot import SebaSnapshot -from ert.storage import open_storage from everest.config import EverestConfig from everest.strings import STORAGE_DIR @@ -161,7 +161,9 @@ def get_internalized_keys(config: EverestConfig, batch_ids: Optional[Set[int]] = internal_keys: Set = set() with open_storage(config.storage_dir, "r") as storage: for batch_id in batch_ids: - ensemble = storage.get_ensemble_by_name("batch_{}".format(batch_id)) + case_name = f"batch_{batch_id}" + experiment = storage.get_experiment_by_name(f"experiment_{case_name}") + ensemble = experiment.get_ensemble_by_name(case_name) if not internal_keys: internal_keys = set(ensemble.get_summary_keyset()) else: @@ -310,9 +312,10 @@ def _load_simulation_data( with open_storage(ens_path, "r") as storage: # pylint: disable=unnecessary-lambda-assignment def load_batch_by_id(): - return storage.get_ensemble_by_name( - f"batch_{batch}" - ).load_all_summary_data() + case_name = f"batch_{batch}" + experiment = storage.get_experiment_by_name(f"experiment_{case_name}") + ensemble = experiment.get_ensemble_by_name(case_name) + return ensemble.load_all_summary_data() batches = {elem[MetaDataColumnNames.BATCH] for elem in metadata} batch_data = [] diff --git a/src/everest/jobs/scripts/recovery_factor b/src/everest/jobs/scripts/recovery_factor index 41c37ab59d3..db4246e5d10 100755 --- a/src/everest/jobs/scripts/recovery_factor +++ b/src/everest/jobs/scripts/recovery_factor @@ -20,8 +20,8 @@ def _build_argument_parser() -> argparse.ArgumentParser: parser.add_argument( "--output_file", required=False, - default="rf_0", - help="the output file, defaults to rf_0", + default="rf", + help="the output file, defaults to rf", ) return parser diff --git a/src/everest/queue_driver/queue_driver.py b/src/everest/queue_driver/queue_driver.py index b4454d4f57c..a6170859f7a 100644 --- a/src/everest/queue_driver/queue_driver.py +++ b/src/everest/queue_driver/queue_driver.py @@ -1,6 +1,7 @@ from typing import Any, List, Optional, Tuple from ert.config import QueueSystem + from everest.config import EverestConfig from everest.config.simulator_config import SimulatorConfig from everest.config_keys import ConfigKeys diff --git a/src/everest/simulator/__init__.py b/src/everest/simulator/__init__.py index 3cf043e5841..cae3abf5dd5 100644 --- a/src/everest/simulator/__init__.py +++ b/src/everest/simulator/__init__.py @@ -1,4 +1,5 @@ from ert.simulator.batch_simulator_context import Status + from everest.simulator.simulator import Simulator JOB_SUCCESS = "Success" diff --git a/src/everest/simulator/simulator.py b/src/everest/simulator/simulator.py index eafe861f317..b7ac429197d 100644 --- a/src/everest/simulator/simulator.py +++ b/src/everest/simulator/simulator.py @@ -4,13 +4,13 @@ from typing import Any, DefaultDict, Dict, List, Mapping, Optional, Tuple, Union import numpy as np +from ert import BatchSimulator, WorkflowRunner +from ert.config import ErtConfig, HookRuntime +from ert.storage import open_storage from numpy import float64 from numpy._typing import NDArray from ropt.evaluator import EvaluatorContext, EvaluatorResult -from ert import BatchSimulator, WorkflowRunner -from ert.config import ErtConfig, HookRuntime -from ert.storage import open_storage from everest.config import EverestConfig from everest.config.control_variable_config import ( ControlVariableConfig, diff --git a/src/everest/suite.py b/src/everest/suite.py index 8458911b566..59f6d72f741 100644 --- a/src/everest/suite.py +++ b/src/everest/suite.py @@ -16,8 +16,6 @@ from ropt.enums import EventType from ropt.plan import OptimizationPlanRunner -from ropt.report import ResultsTable -from ropt.results import convert_to_maximize from seba_sqlite import SqliteStorage import everest @@ -367,9 +365,9 @@ def _simulation_callback(self, *args, **_): def _ropt_callback(self, event, optimizer, simulator): logging.getLogger(EVEREST).debug("Optimization callback called") - if ( - self._max_batch_num is not None - and simulator.number_of_evaluated_batches >= self._max_batch_num + if self._config.optimization.max_batch_num is not None and ( + simulator.number_of_evaluated_batches + >= self._config.optimization.max_batch_num ): self._max_batch_num_reached = True logging.getLogger(EVEREST).info("Maximum number of batches reached") @@ -390,29 +388,13 @@ def start_optimization(self): This method is not thread safe. Multiple overlapping executions of this method will probably lead to a crash """ - if self._monitor_thread is not None: - raise RuntimeError("Expected monitoring thread to be None!") - - # max_batch_num is tracked by Everest itself: - self._max_batch_num = self._config.optimization.max_batch_num + assert self._monitor_thread is None # Initialize the Everest simulator: simulator = Simulator(self.config, callback=self._simulation_callback) # Initialize the ropt optimizer: - optimizer = OptimizationPlanRunner( - enopt_config=everest2ropt(self.config), - evaluator=simulator, - seed=self._config.environment.random_seed, - ) - - # Configure restarting: - if self.config.optimization.restart is not None: - optimizer.repeat( - iterations=self.config.optimization.restart.max_restarts + 1, - restart_from=self.config.optimization.restart.restart_from, - counter_var="restart", - ).add_metadata({"restart": "$restart"}) + optimizer = self._configure_optimizer(simulator) # Before each batch evaluation we check if we should abort: optimizer.add_observer( @@ -428,9 +410,6 @@ def start_optimization(self): # is retained for now via the seba_sqlite package. seba_storage = SqliteStorage(optimizer, self.config.optimization_output_dir) - # Initialize tabular reporters - self._init_reporters(optimizer, self.config.optimization_output_dir) - # Run the optimization: exit_code = optimizer.run().exit_code @@ -457,50 +436,51 @@ def __repr__(self): self.config, sort_keys=True, indent=2 ) - def _init_reporters(self, optimizer, ropt_output_folder): - # Initialize the tabular reporters, which writes the optimization results - # as a table with fixed-width columns: - results_report = ResultsTable( + def _configure_optimizer(self, simulator: Simulator) -> OptimizationPlanRunner: + optimizer = OptimizationPlanRunner( + enopt_config=everest2ropt(self.config), + evaluator=simulator, + seed=self._config.environment.random_seed, + ) + + # Configure restarting: + if self.config.optimization.restart is not None: + optimizer.repeat( + iterations=self.config.optimization.restart.max_restarts + 1, + restart_from=self.config.optimization.restart.restart_from, + metadata_var="restart", + ) + + # Initialize output tables. `min_header_len` is set to ensure that all + # tables have the same number of header lines, simplifying code that + # reads them as fixed width tables. `maximize` is set because ropt + # reports minimization results, while everest wants maximization + # results, necessitating a conversion step. + ropt_output_folder = Path(self.config.optimization_output_dir) + optimizer.add_table( columns=RESULT_COLUMNS, - path=Path(ropt_output_folder) / "results.txt", + path=ropt_output_folder / "results.txt", min_header_len=MIN_HEADER_LEN, + maximize=True, ) - gradient_report = ResultsTable( + optimizer.add_table( columns=GRADIENT_COLUMNS, - path=Path(ropt_output_folder) / "gradients.txt", + path=ropt_output_folder / "gradients.txt", table_type="gradients", min_header_len=MIN_HEADER_LEN, + maximize=True, ) - simulations_report = ResultsTable( + optimizer.add_table( columns=SIMULATION_COLUMNS, - path=Path(ropt_output_folder) / "simulations.txt", + path=ropt_output_folder / "simulations.txt", min_header_len=MIN_HEADER_LEN, + maximize=True, ) - perturbations_report = ResultsTable( + optimizer.add_table( columns=PERTURBATIONS_COLUMNS, - path=Path(ropt_output_folder) / "perturbations.txt", + path=ropt_output_folder / "perturbations.txt", table_type="gradients", min_header_len=MIN_HEADER_LEN, + maximize=True, ) - - # Results passed from ropt are for minimization of the negative of the - # objective function. We want results that correspond to the - # maximization. This event handler converts the results and forwards - # them to the reporter: - def _handle_reporter_event(event, reporters): - results = tuple(convert_to_maximize(result) for result in event.results) - for reporter in reporters: - reporter.add_results(event.config, results) - - optimizer.add_observer( - EventType.FINISHED_EVALUATION, - partial( - _handle_reporter_event, - reporters=[ - results_report, - gradient_report, - simulations_report, - perturbations_report, - ], - ), - ) + return optimizer diff --git a/src/everest/util/__init__.py b/src/everest/util/__init__.py index 840ca2287b3..88b04ea4101 100644 --- a/src/everest/util/__init__.py +++ b/src/everest/util/__init__.py @@ -2,9 +2,9 @@ import logging import os +from ert.shared.version import version as ert_version from ropt.version import version as ropt_version -from ert.shared.version import version as ert_version from everest.plugins.hook_manager import EverestPluginManager from everest.strings import DATE_FORMAT, DEFAULT_LOGGING_FORMAT from everest.util.async_run import async_run # noqa diff --git a/src/everest/version.py b/src/everest/version.py index f324f2f3b56..460609468fb 100644 --- a/src/everest/version.py +++ b/src/everest/version.py @@ -3,7 +3,6 @@ TYPE_CHECKING = False if TYPE_CHECKING: from typing import Tuple, Union - VERSION_TUPLE = Tuple[Union[int, str], ...] else: VERSION_TUPLE = object @@ -13,5 +12,5 @@ __version_tuple__: VERSION_TUPLE version_tuple: VERSION_TUPLE -__version__ = version = "1.1.1.dev8+g36124081" -__version_tuple__ = version_tuple = (1, 1, 1, "dev8", "g36124081") +__version__ = version = '1.1.1.dev19+gba468db7.d20240909' +__version_tuple__ = version_tuple = (1, 1, 1, 'dev19', 'gba468db7.d20240909') diff --git a/src/ieverest/bin/ieverest_script.py b/src/ieverest/bin/ieverest_script.py index 74f9f062636..a46f007b599 100755 --- a/src/ieverest/bin/ieverest_script.py +++ b/src/ieverest/bin/ieverest_script.py @@ -4,10 +4,10 @@ import sys from argparse import ArgumentParser +from everest.util import version_info from qtpy.QtCore import Qt from qtpy.QtWidgets import QApplication -from everest.util import version_info from ieverest import IEverest diff --git a/src/ieverest/config_widget.py b/src/ieverest/config_widget.py index a505e3dd4b0..cd5ea53652c 100644 --- a/src/ieverest/config_widget.py +++ b/src/ieverest/config_widget.py @@ -1,9 +1,9 @@ from io import StringIO +from everest.config import EverestConfig from qtpy.QtWidgets import QWidget from ruamel.yaml import YAML -from everest.config import EverestConfig from ieverest import utils diff --git a/src/ieverest/ieverest.py b/src/ieverest/ieverest.py index 080e3246aca..fecae9dfb09 100644 --- a/src/ieverest/ieverest.py +++ b/src/ieverest/ieverest.py @@ -1,7 +1,3 @@ -from pydantic import ValidationError -from qtpy.QtCore import QObject, QThread, Signal -from qtpy.QtWidgets import QFileDialog, QMessageBox, qApp - from ert.config import ErtConfig from ert.storage import open_storage from everest.config import EverestConfig, validation_utils @@ -23,6 +19,10 @@ from everest.simulator import Status from everest.strings import OPT_PROGRESS_ID, SIM_PROGRESS_ID from everest.util import configure_logger, makedirs_if_needed +from pydantic import ValidationError +from qtpy.QtCore import QObject, QThread, Signal +from qtpy.QtWidgets import QFileDialog, QMessageBox, qApp + from ieverest import settings from ieverest.io import QtDialogsOut, QtStatusBarOut from ieverest.main_window import MainWindow diff --git a/src/ieverest/main_window.py b/src/ieverest/main_window.py index 1804a65b483..d98a5eaa73a 100644 --- a/src/ieverest/main_window.py +++ b/src/ieverest/main_window.py @@ -1,5 +1,7 @@ from functools import partial +from everest import __version__ as everest_version +from everest.config import EverestConfig from qtpy.QtCore import Qt, Signal from qtpy.QtGui import QIcon from qtpy.QtWidgets import ( @@ -13,8 +15,6 @@ QWidget, ) -from everest import __version__ as everest_version -from everest.config import EverestConfig from ieverest.config_widget import ConfigWidget from ieverest.monitor_widget import MonitorWidget from ieverest.utils import load_ui diff --git a/src/ieverest/monitor_widget.py b/src/ieverest/monitor_widget.py index ff39259f892..dc904ce509d 100644 --- a/src/ieverest/monitor_widget.py +++ b/src/ieverest/monitor_widget.py @@ -2,11 +2,11 @@ import pandas as pd from dateutil import parser +from everest.config import EverestConfig +from everest.simulator import JOB_FAILURE, JOB_SUCCESS from qtpy.QtCore import Signal from qtpy.QtWidgets import QComboBox, QVBoxLayout, QWidget -from everest.config import EverestConfig -from everest.simulator import JOB_FAILURE, JOB_SUCCESS from ieverest.utils import load_ui, remove_layout_item from ieverest.widgets import BatchStatusWidget, PlotWidget from ieverest.widgets.batch_status_widget import get_job_status diff --git a/src/ieverest/widgets/batch_status_widget.py b/src/ieverest/widgets/batch_status_widget.py index 9b48f6feecc..24a8a6f1d7d 100644 --- a/src/ieverest/widgets/batch_status_widget.py +++ b/src/ieverest/widgets/batch_status_widget.py @@ -1,8 +1,8 @@ +from everest.simulator import JOB_FAILURE, JOB_RUNNING, JOB_SUCCESS, JOB_WAITING from qtpy.QtCore import QSize, Qt from qtpy.QtGui import QIcon, QPixmap from qtpy.QtWidgets import QLabel, QSizePolicy, QSpacerItem, QStyle, QWidget, qApp -from everest.simulator import JOB_FAILURE, JOB_RUNNING, JOB_SUCCESS, JOB_WAITING from ieverest.utils import load_ui diff --git a/src/ieverest/widgets/export_data_dialog.py b/src/ieverest/widgets/export_data_dialog.py index 078f0132252..fa89f69afc3 100644 --- a/src/ieverest/widgets/export_data_dialog.py +++ b/src/ieverest/widgets/export_data_dialog.py @@ -1,10 +1,10 @@ import os -from qtpy.QtWidgets import QDialog, QFileDialog - from everest.config import EverestConfig from everest.config.export_config import ExportConfig from everest.export import export +from qtpy.QtWidgets import QDialog, QFileDialog + from ieverest.utils import APP_OUT_DIALOGS, APP_OUT_STATUS_BAR, app_output, load_ui