Add DRL project starter code.

.gitignore

@@ -1,5 +1,9 @@
 # Prerequisites
 *.d
+__pycache__
+.vscode
+dynamicCode
+linux64GccDPInt32Opt
 
 # Compiled Object files
 *.slo

2022-07/ofml-drl/Allwclean

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+wclean all src/cpp

2022-07/ofml-drl/Allwmake

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+if [ -z $OFML_DRL_TORCH ]; 
+then 
+    echo "Torch not configured, edit variables in setup-etc, then 'source setup-etc'"
+    exit 1
+fi
+
+if [ ! -d "libtorch" ]; 
+then 
+    wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.12.0%2Bcpu.zip && \
+    unzip libtorch-cxx11-abi-shared-with-deps-1.12.0+cpu.zip
+fi
+
+wmake all src/cpp

2022-07/ofml-drl/README.md

@@ -0,0 +1,65 @@
+# DRL for flow control in OpenFOAM
+
+This code is re-factored (and currently incomplete) version of the following repositories:
+
+- student projects by Darshan Thummar ([link](https://github.com/darshan315/flow_past_cylinder_by_DRL)) and Fabian Gabriel ([link](https://github.com/FabianGabriel/Active_flow_control_past_cylinder_using_DRL))
+- exercise 11 of [this lecture](https://github.com/AndreWeiner/ml-cfd-lecture/blob/main/notebooks/exercise_10_11.ipynb)
+
+A brief theoretical overview may be found [here](https://andreweiner.github.io/ml-cfd-slides/lecture_10_11.html).
+
+## Dependencies
+
+The main dependencies are:
+- OpenFOAM-v2206; standard package installation is fine
+- PyTorch-1.12.0; ideally, create a virtual environment as shown below
+
+Creating and using a virtual Python environment:
+- install *venv*; on Ubuntu 20.04 and above, run `sudo apt install python3.8-venv`
+- set up a new environment as show below:
+```
+python3 -m venv pydrl
+source pydrl/bin/activate
+pip install -r requirements.txt
+deactivate
+```
+
+For running the unittests, you also need *pytest*.
+
+
+## Compiling the C++ code
+
+Source the environment variables and compile:
+
+```
+source setup-env
+./Allwmake
+```
+
+## Running the Python code
+
+### Unittests
+
+```
+source setup-env
+source pydrl/bin/activate
+pytest -s src/python
+```
+
+### Training
+
+The main training loop is implemented in *run_training.py*. However, the `fill()` method of the `LocalBuffer` class is not yet implemented. Moreover, the `SlurmBuffer` class for execution on the cluster is missing. Once the `Buffer` is complete, the training can be started by running:
+
+```
+python3 -m run_training.py
+```
+
+## Tasks for the hackathon
+
+1. implement the `LocalBuffer.fill()` method; [hint](https://github.com/AndreWeiner/ml-cfd-lecture/blob/main/test_cases/drl_control_cylinder/env_local.py)
+2. implement the `SlurmBuffer` class analogously; [hint](https://github.com/FabianGabriel/Active_flow_control_past_cylinder_using_DRL/blob/main/DRL_py_beta/env_cluster.py)
+3. implement one or more of the ideas below or your own ideas
+
+- the main challenge in DRL with CFD is the relatively large cost of the *environment*; the training might be accelerated as follows:
+  - buffer size: a larger buffer size requires more computational resources but stabilizes the training; find and implement a criterion to adjust the buffer size during training
+  - trajectory length: if the trajectory is shorter, the training is accelerated, but there might be too little exploration to learn; find and implement a criterion to adjust the trajectory length during training
+- one important question for applications is knowing the applicability of a policy under changed flow conditions, e.g., are we extrapolating in the feature space? Use a Bayesian network as policy to provide uncertainty bounds for the control

2022-07/ofml-drl/requirements.txt

@@ -0,0 +1,3 @@
+torch==1.12.0
+pandas
+numpy

2022-07/ofml-drl/run_training.py

@@ -0,0 +1,69 @@
+
+from python.buffer import LocalBuffer
+from python.agent import PPOAgent
+from python.environment import RotatingCylinder2D
+import pickle
+from shutil import copytree
+from os import makedirs
+import sys
+
+sys.path.insert(0, "src")
+
+
+def print_statistics(actions, rewards):
+    rt = [r.mean().item() for r in rewards]
+    at_mean = [a.mean().item() for a in actions]
+    at_std = [a.std().item() for a in actions]
+    print("Reward mean/min/max: ", sum(rt)/len(rt), min(rt), max(rt))
+    print("Mean action mean/min/max: ", sum(at_mean) /
+          len(at_mean), min(at_mean), max(at_mean))
+    print("Std. action mean/min/max: ", sum(at_std) /
+          len(at_std), min(at_std), max(at_std))
+
+
+def main():
+
+    # setting
+    training_path = "test_training"
+    epochs = 10
+    buffer_size = 2
+    n_runners = 2
+
+    # create a directory for training
+    makedirs(training_path, exist_ok=True)
+
+    # make a copy of the base environment
+    copytree(join("test_cases", "rotatingCylinder2D"),
+             join(training_path, "base"), dirs_exist_ok=True)
+    env = RotatingCylinder2D()
+    env.path = join(training_path, "base")
+    env.end_time = 6
+    env.reset()
+
+    # create a trajectory buffer
+    buffer = LocalBuffer(training_path, env, buffer_size, n_runners)
+
+    # create PPO agent
+    agent = PPOAgent(env.n_states, env.n_actions, -
+                     env.action_bounds, env.action_bounds)
+
+    # begin training
+    for e in range(epochs):
+        buffer.fill()
+        states, actions, rewards, log_p = buffer.sample()
+        print_statistics(actions, rewards)
+        with open(join(training_path, f"observations_e{e}.pkl")) as f:
+            pickle.dump((states, actions, rewards, log_p), f)
+        agent.update(states, actions, rewards, log_p)
+        buffer.reset()
+        buffer.update_policy(agent.trace_policy())
+        agent.save(join(training_path, f"policy_{e}.pt"),
+                   join(training_path, f"value_{e}.pt"))
+
+    # save statistics
+    with open(join(training_path, "training_history.pkl")) as f:
+        pickle.dump(agent.history, f)
+
+
+if __name__ == "__main__":
+    main()

2022-07/ofml-drl/setup-env

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# set environment variables
+export OFML_DRL="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export OFML_DRL_TORCH=$OFML_DRL/libtorch

2022-07/ofml-drl/src/cpp/agentRotatingWallVelocity/Make/files

@@ -0,0 +1,2 @@
+agentRotatingWallVelocityFvPatchVectorField.C
+LIB = $(FOAM_USER_LIBBIN)/libAgentRotatingWallVelocity

2022-07/ofml-drl/src/cpp/agentRotatingWallVelocity/Make/options

@@ -0,0 +1,14 @@
+EXE_INC = \
+    -std=c++2a \
+    -Wno-deprecated-declarations -Wno-old-style-cast -Wno-redundant-move \
+    -I$(LIB_SRC)/finiteVolume/lnInclude \
+    -I$(OFML_DRL_TORCH)/include \
+    -I$(OFML_DRL_TORCH)/include/torch/csrc/api/include
+
+LIB_LIBS = \
+    -lfiniteVolume \
+    -rdynamic \
+    -Wl,-rpath,$(OFML_DRL_TORCH)/lib $(OFML_DRL_TORCH)/lib/libtorch.so $(OFML_DRL_TORCH)/lib/libc10.so \
+    -Wl,--no-as-needed,$(OFML_DRL_TORCH)/lib/libtorch_cpu.so \
+    -Wl,--as-needed $(OFML_DRL_TORCH)/lib/libc10.so \
+    -Wl,--no-as-needed,$(OFML_DRL_TORCH)/lib/libtorch.so