Issue492 simulation compare (#493)

Added module to compare simulation performance.
For #492

Refactored regression tests for Dymola to allow specifying a time out for each tests, and set the default time out to 300 seconds
For #495

Co-authored-by: JayHuLBL <[email protected]>

Author: mwetter

Makefile

@@ -53,6 +53,9 @@ unittest_development_regressiontest:
 unittest_development_Validator:
 	python3 buildingspy/tests/test_development_Validator.py
 
+unittest_development_Comparator:
+	python3 buildingspy/tests/test_development_Comparator.py
+
 unittest_examples_dymola:
 	python3 buildingspy/tests/test_examples_dymola.py
 

buildingspy/CHANGES.txt

@@ -3,6 +3,11 @@ BuildingsPy Changelog
 
 Version 4.x.x, xxx -- Release 4.0
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+- Added class buildingspy.development.simulationCompare that compares
+  the simulation performance across tools or git branches.
+  (https://github.com/lbl-srg/BuildingsPy/issues/492)
+- Refactored regression tests for Dymola to allow specifying a time out for each tests, and set the default time out to 300 seconds.
+  (https://github.com/lbl-srg/BuildingsPy/issues/495)
 - Add option to exclude simulation of models from Dymola CI tests.
   (https://github.com/lbl-srg/BuildingsPy/pull/486)
 

buildingspy/development/__init__.py

@@ -1,10 +1,11 @@
 """
 This module contains the classes
 
-- *refactor*, a module that assists in refactoring Modelica classes,
-- *Tester* that runs the unit tests of the `Buildings` library,
-- *Validator* that validates the html code of the info section of the `.mo` files, and
-- *IBPSA* that synchronizes Modelica libraries with the `IBPSA` library.
-- *ErrorDictionary* that contains information about possible error strings.
+- :func:`refactor <buildingspy.development.refactor>`, a module that assists in refactoring Modelica classes,
+- :func:`Tester <buildingspy.development.regressiontest.Tester>` that runs the unit tests of the `Buildings` library,
+- :func:`Validator <buildingspy.development.validator.Validator>` that validates the html code of the info section of the `.mo` files, and
+- :func:`IBPSA <buildingspy.development.merger.IBPSA>` that synchronizes Modelica libraries with the `IBPSA` library.
+- :func:`ErrorDictionary <buildingspy.development.error_dictionary.ErrorDictionary>` that contains information about possible error strings.
+- :func:`Comparator <buildingspy.development.simulationCompare.Comparator>` that compares the simulation performance across simulation tools or git branches.
 
 """

buildingspy/development/dymola_run.template

@@ -0,0 +1,83 @@
+def _add_exception(return_dict, e, cmd):
+  import subprocess
+
+  return_dict['success'] = False
+
+#  if isinstance(e, subprocess.CalledProcessError):
+#    # Check if simulation terminated, and if so, get the error
+#    return_dict['stdout'] = e.output.decode("utf-8")
+#    output = return_dict['stdout']
+#    for line in output.split('\n'):
+#      if 'terminated' in line:
+#        # Found terminated string. Cut everything after the '|' character that OpenModelica writes.
+#        idx=line.rfind('|')
+#        msg=line[idx+1:].strip()
+#        # The solver terminated. Add this information to a custom exception message.
+#        return_dict['exception'] = f"'{' '.join(cmd)}' caused '{msg}'."
+#        pass
+
+  if not 'exception' in return_dict:
+    # Did not find 'terminated' in message, handle exception as usual
+    return_dict['exception'] = '{}: {}'.format(type(e).__name__, e)
+
+
+def _run_process(return_dict, cmd, worDir, timeout):
+  import subprocess
+
+  output = subprocess.check_output(
+    cmd,
+    cwd = worDir,
+    timeout=timeout,
+    stderr=subprocess.STDOUT,
+    shell=False)
+
+  return_dict['success'] = True
+  if 'stdout' in return_dict:
+     return_dict['stdout'] += output.decode("utf-8")
+  else:
+     return_dict['stdout'] = output.decode("utf-8")
+  return
+
+def _simulate(model, timeout):
+  import os
+  import subprocess
+
+  worDir = "{{ working_directory }}"
+  return_dict = {}
+
+  try:
+    cmd = {{ cmd }}
+    return_dict['cmd'] = ' '.join(cmd)
+    output = _run_process(return_dict, cmd, worDir, timeout)
+
+  except Exception as e:
+    _add_exception(return_dict, e, cmd)
+  return return_dict
+
+def run():
+  import os
+  import json
+  import traceback
+  import sys
+
+  timeout = {{ time_out }}
+  model = "{{ model }}"
+  result = {"model": model,
+            "working_directory": "{{ working_directory }}",
+            "simulation": {"success": False}}
+
+  # Log file
+  log_file = "{}_buildingspy.json".format(model.replace(".", "_"))
+  try:
+    os.remove(log_file)
+  except OSError:
+    pass
+
+  # Simulate model
+  result["simulation"] = _simulate(model, timeout)
+
+  with open(log_file, "w") as log:
+    log.write("{}\n".format(json.dumps(result, indent=4, sort_keys=False)) )
+
+if __name__=="__main__":
+    run()