Fix #82 Remove Docker images when a configuration is tested

camp/data/manage_images.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+#
+# Generated by CAMP. Edit carefully\n"
+#
+# Build all images and set the appropriate tags\n"
+#
+
+set -e
+
+BUILD=false
+CLEANUP=false 
+
+parse_arguments () {
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            -b|--build)
+                BUILD="true"
+                shift 1
+                ;;
+            -c|--cleanup)
+                CLEANUP="true"     
+                shift 1
+                ;;
+            *)
+                printf "Error: Unknown option '$1'.\n"
+                printf "${USAGE}\n"
+                exit 1
+        esac
+    done
+
+}
+
+
+# Remove the obselete images created for this configuration
+# See Issue #82 
+remove_obselete_images () {
+    docker rmi {obselete_images}
+    docker image prune --force --filter "label=stage=intermediate"
+}
+
+
+build_images () {
+    {build_commands}
+}
+
+
+# Script entry point
+
+parse_arguments $*
+
+if [[ "${BUILD}" == "true" ]]
+then
+    printf "building images\n"
+    build_images
+else
+    if [[ "${CLEANUP}" == "true" ]]
+    then
+        printf "Cleaning up\n"
+        remove_obselete_images
+    fi
+fi
+

camp/execute/engine.py

@@ -240,7 +240,7 @@ def _build_images(self, path):
         working_directory = join_paths(path, "images")
         self._shell.execute(self._BUILD_IMAGES, working_directory)
 
-    _BUILD_IMAGES = "bash build_images.sh"
+    _BUILD_IMAGES = "bash build_images.sh --build"
 
 
     def _start_services(self, path):
@@ -341,7 +341,14 @@ def _parse_test_reports(self, path):
     def _stop_services(self, path):
         self._shell.execute(self._STOP_SERVICES, path)
 
-    _STOP_SERVICES = "docker-compose down --volumes --rmi all"
+        working_directory = join_paths(path, "images")
+        self._shell.execute("bash build_images.sh --cleanup", working_directory)
+
+
+    # We use the option '--rmi local' to avoid deleting the image
+    # created by the script 'build_images.sh'. These images will be
+    # deleted by the script afterwards during: 'sh build_images.sh --cleanup'
+    _STOP_SERVICES = "docker-compose down --volumes --rmi local"
 
 
 

camp/realize.py

@@ -17,6 +17,8 @@
 from os.path import exists, isdir, isfile, join as join_paths, \
      normpath, split as split_path, relpath
 
+from pkgutil import get_data
+
 from re import escape, sub, subn
 
 from shutil import copyfile, copytree, move, rmtree
@@ -73,8 +75,7 @@ def build(self, configuration, input_directory=None, output_directory=None):
         self._copy_orchestration_files()
         for each_instance in configuration.instances:
             self._copy_template_for(each_instance)
-            if each_instance.feature_provider:
-                self._adjust_docker_file(each_instance)
+            self._adjust_docker_file(each_instance)
             self._realize_component(each_instance)
             self._realize_variables(each_instance)
         self._adjust_docker_compose_file(configuration)
@@ -112,13 +113,12 @@ def _adjust_docker_compose_file(self, configuration):
             with open(orchestration, "r") as source:
                 content = source.read()
                 for each_instance in configuration.instances:
+                    # Issue 82: Here we replace "build" clauses by
+                    # "images" clauses to avoid generating additional
+                    # dangling images
                     content, count = subn(r"build:\s*\./" + each_instance.definition.name,
-                                          "build: ./images/" + each_instance.name,
+                                          "image: " + self._docker_tag_for(each_instance),
                                           content)
-                    # Investigating Issue #55
-                    # if count == 0:
-                    #     raise RuntimeError("Component " + each_instance.definition.name \
-                    #                        + " cannot be found in the dockerfile")
 
             with open(orchestration, "w") as target:
                 target.write(content)
@@ -165,22 +165,26 @@ def _file_for(self, instance, resource):
 
     def _adjust_docker_file(self, instance):
         self._record_dependency_of(instance)
-        host = instance.feature_provider.definition.implementation
-        kind = type(host)
-        if kind == DockerImage:
-            self._replace_in(
-                self._docker_file_for(instance),
-                instance,
-                self.REGEX_FROM,
-                "FROM " + host.docker_image)
-        elif kind == DockerFile:
-            self._replace_in(
-                self._docker_file_for(instance),
-                instance,
-                self.REGEX_FROM,
-                "FROM %s" % self._docker_tag_for(instance.feature_provider))
+        if not instance.feature_provider:
+            pass
+            # Nothing to change on the dockerfile
         else:
-            raise RuntimeError("Component implement '%s' not supported yet" \
+            host = instance.feature_provider.definition.implementation
+            kind = type(host)
+            if kind == DockerImage:
+                self._replace_in(
+                    self._docker_file_for(instance),
+                    instance,
+                    self.REGEX_FROM,
+                    "FROM " + host.docker_image)
+            elif kind == DockerFile:
+                self._replace_in(
+                    self._docker_file_for(instance),
+                    instance,
+                    self.REGEX_FROM,
+                    "FROM %s" % self._docker_tag_for(instance.feature_provider))
+            else:
+                raise RuntimeError("Component implement '%s' not supported yet" \
                                % kind.__name__)
 
     # Issue 78 about Multi-stages build.
@@ -307,45 +311,56 @@ def _docker_tag_for(instance):
 
 
     def _record_dependency_of(self, instance):
-        if instance.feature_provider in self._images:
-            index = self._images.index(instance.feature_provider)
-            self._images.insert(index+1, instance)
-        elif instance in self._images:
-            index = self._images.index(instance)
-            self._images.insert(index, instance.feature_provider)
+        if instance not in self._images:
+            if instance.feature_provider:
+                if instance.feature_provider in self._images:
+                    index = self._images.index(instance.feature_provider)
+                    self._images.insert(index+1, instance)
+                else:
+                    self._images.append(instance.feature_provider)
+                    self._images.append(instance)
+            else:
+                self._images.append(instance)
         else:
-            self._images.append(instance.feature_provider)
-            self._images.append(instance)
-
+            if instance.feature_provider:
+                if instance.feature_provider not in self._images:
+                    index = self._images.index(instance)
+                    self._images.insert(index, instance.feature_provider)
 
 
     def _generate_build_script(self):
         build_commands = []
+        obselete_images = []
         for each_instance in self._images:
             if isinstance(each_instance.definition.implementation, DockerFile):
                 tag = self._docker_tag_for(each_instance)
+                obselete_images.append(tag)
                 folder = "./" + each_instance.name
                 command = self.BUILD_COMMAND.format(folder=folder, tag=tag)
                 build_commands.append(command)
 
-        build_script = self._build_script()
-        with open(build_script, "w") as stream:
-            content = self.BUILD_SCRIPT_TEXT.format("\n".join(build_commands))
-            stream.write(content)
+        script = self._build_script()
+        with open(script, "w") as stream:
+            body = self._fetch_script_template()
+            body = sub(self.BUILD_COMMAND_MARKER,
+                       "\n\t".join(build_commands),
+                       body)
+            body = sub(self.OBSELETE_IMAGES_MARKER,
+                       " ".join(obselete_images),
+                       body)
+            stream.write(body)
 
+    BUILD_COMMAND_MARKER = "{build_commands}"
 
-    BUILD_COMMAND = "docker build --no-cache -t {tag} {folder}"
+    OBSELETE_IMAGES_MARKER = "{obselete_images}"
 
-    def _build_script(self):
-        return join_paths(self._image_directory, "build_images.sh")
+    def _fetch_script_template(self):
+        return get_data('camp', 'data/manage_images.sh').decode("utf-8")
 
 
-    BUILD_SCRIPT_TEXT = ("#!/bin/bash\n"
-                         "#\n"
-                         "# Generated by CAMP. Edit carefully\n"
-                         "#\n"
-                         "# Build all images and set the appropriate tags\n"
-                         "#\n"
-                         "set -e\n"
-                         "{0}\n"
-                         "echo 'All images ready.'\n")
+    # Issue 82: The "--force-rm" option avoids generating many
+    # dangling images and consuming a lot of disk space
+    BUILD_COMMAND = "docker build --force-rm --no-cache -t {tag} {folder}"
+
+    def _build_script(self):
+        return join_paths(self._image_directory, "build_images.sh")

samples/java/template/greetings/Dockerfile

@@ -11,7 +11,8 @@
 # Step 1: Build the WAR file
 FROM openjdk:8-jdk-stretch as builder
 
-LABEL maintainer "[email protected]"
+LABEL maintainer="[email protected]"
+LABEL stage="intermediate"
 
 RUN apt-get update && \
 	apt-get install -y --no-install-recommends  \

samples/stamp/ow2/template/lutece/Dockerfile

@@ -13,6 +13,8 @@ ARG site=site-forms-demo-1.0.0-SNAPSHOT
 # A first container to build the lutece web app
 FROM debian:stretch as builder
 
+LABEL stage="intermediate"
+
 RUN apt-get update && apt-get dist-upgrade -y && \
         apt-get install -y --no-install-recommends \
         mysql-client \

setup.py

@@ -31,7 +31,10 @@
       packages=find_packages(exclude=["tests*", "tests.*"]),
       include_package_data = True,
       package_data = {
-          "camp": ["data/metamodel.yml"]
+          "camp": [
+              "data/metamodel.yml",
+              "data/manage_images.sh"
+          ]
       },
       license="MIT",
       test_suite="tests",

tests/realize/test_docker.py

@@ -117,9 +117,9 @@ def test_when_the_stack_has_more_than_two_components(self):
         self.build(configuration)
 
         expected_command_order = (
-            "docker build --no-cache -t camp-jdk_0 ./jdk_0\n"
-            "docker build --no-cache -t camp-tomcat_0 ./tomcat_0\n"
-            "docker build --no-cache -t camp-server_0 ./server_0\n"
+            "docker build --force-rm --no-cache -t camp-jdk_0 ./jdk_0\n"
+            "\tdocker build --force-rm --no-cache -t camp-tomcat_0 ./tomcat_0\n"
+            "\tdocker build --force-rm --no-cache -t camp-server_0 ./server_0\n"
         )
 
         self.assert_generated(

tests/realize/test_operators.py

@@ -206,5 +206,5 @@ def test_select_resource_before_substitutions_take_place(self):
 
             self.assertIn("nginx_variable=something_else",
                           each_configuration.content_of("docker-compose.yml"))
-            self.assertIn("build: ./images/nginx_0",
+            self.assertIn("image: camp-nginx_0",
                           each_configuration.content_of("docker-compose.yml"))