diff --git a/camp/data/manage_images.sh b/camp/data/manage_images.sh new file mode 100755 index 00000000..f985c5a0 --- /dev/null +++ b/camp/data/manage_images.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Generated by CAMP. Edit carefully\n" +# +# Build all images and set the appropriate tags\n" +# + +set -e + +BUILD=false +CLEANUP=false + +parse_arguments () { + while [ $# -gt 0 ]; do + case "$1" in + -b|--build) + BUILD="true" + shift 1 + ;; + -c|--cleanup) + CLEANUP="true" + shift 1 + ;; + *) + printf "Error: Unknown option '$1'.\n" + printf "${USAGE}\n" + exit 1 + esac + done + +} + + +# Remove the obselete images created for this configuration +# See Issue #82 +remove_obselete_images () { + docker rmi {obselete_images} + docker image prune --force --filter "label=stage=intermediate" +} + + +build_images () { + {build_commands} +} + + +# Script entry point + +parse_arguments $* + +if [[ "${BUILD}" == "true" ]] +then + printf "building images\n" + build_images +else + if [[ "${CLEANUP}" == "true" ]] + then + printf "Cleaning up\n" + remove_obselete_images + fi +fi + diff --git a/camp/execute/engine.py b/camp/execute/engine.py index bb817e37..cf3c195f 100644 --- a/camp/execute/engine.py +++ b/camp/execute/engine.py @@ -240,7 +240,7 @@ def _build_images(self, path): working_directory = join_paths(path, "images") self._shell.execute(self._BUILD_IMAGES, working_directory) - _BUILD_IMAGES = "bash build_images.sh" + _BUILD_IMAGES = "bash build_images.sh --build" def _start_services(self, path): @@ -341,7 +341,14 @@ def _parse_test_reports(self, path): def _stop_services(self, path): self._shell.execute(self._STOP_SERVICES, path) - _STOP_SERVICES = "docker-compose down --volumes --rmi all" + working_directory = join_paths(path, "images") + self._shell.execute("bash build_images.sh --cleanup", working_directory) + + + # We use the option '--rmi local' to avoid deleting the image + # created by the script 'build_images.sh'. These images will be + # deleted by the script afterwards during: 'sh build_images.sh --cleanup' + _STOP_SERVICES = "docker-compose down --volumes --rmi local" diff --git a/camp/realize.py b/camp/realize.py index d01e1689..d27e2ef6 100644 --- a/camp/realize.py +++ b/camp/realize.py @@ -17,6 +17,8 @@ from os.path import exists, isdir, isfile, join as join_paths, \ normpath, split as split_path, relpath +from pkgutil import get_data + from re import escape, sub, subn from shutil import copyfile, copytree, move, rmtree @@ -73,8 +75,7 @@ def build(self, configuration, input_directory=None, output_directory=None): self._copy_orchestration_files() for each_instance in configuration.instances: self._copy_template_for(each_instance) - if each_instance.feature_provider: - self._adjust_docker_file(each_instance) + self._adjust_docker_file(each_instance) self._realize_component(each_instance) self._realize_variables(each_instance) self._adjust_docker_compose_file(configuration) @@ -112,13 +113,12 @@ def _adjust_docker_compose_file(self, configuration): with open(orchestration, "r") as source: content = source.read() for each_instance in configuration.instances: + # Issue 82: Here we replace "build" clauses by + # "images" clauses to avoid generating additional + # dangling images content, count = subn(r"build:\s*\./" + each_instance.definition.name, - "build: ./images/" + each_instance.name, + "image: " + self._docker_tag_for(each_instance), content) - # Investigating Issue #55 - # if count == 0: - # raise RuntimeError("Component " + each_instance.definition.name \ - # + " cannot be found in the dockerfile") with open(orchestration, "w") as target: target.write(content) @@ -165,22 +165,26 @@ def _file_for(self, instance, resource): def _adjust_docker_file(self, instance): self._record_dependency_of(instance) - host = instance.feature_provider.definition.implementation - kind = type(host) - if kind == DockerImage: - self._replace_in( - self._docker_file_for(instance), - instance, - self.REGEX_FROM, - "FROM " + host.docker_image) - elif kind == DockerFile: - self._replace_in( - self._docker_file_for(instance), - instance, - self.REGEX_FROM, - "FROM %s" % self._docker_tag_for(instance.feature_provider)) + if not instance.feature_provider: + pass + # Nothing to change on the dockerfile else: - raise RuntimeError("Component implement '%s' not supported yet" \ + host = instance.feature_provider.definition.implementation + kind = type(host) + if kind == DockerImage: + self._replace_in( + self._docker_file_for(instance), + instance, + self.REGEX_FROM, + "FROM " + host.docker_image) + elif kind == DockerFile: + self._replace_in( + self._docker_file_for(instance), + instance, + self.REGEX_FROM, + "FROM %s" % self._docker_tag_for(instance.feature_provider)) + else: + raise RuntimeError("Component implement '%s' not supported yet" \ % kind.__name__) # Issue 78 about Multi-stages build. @@ -307,45 +311,56 @@ def _docker_tag_for(instance): def _record_dependency_of(self, instance): - if instance.feature_provider in self._images: - index = self._images.index(instance.feature_provider) - self._images.insert(index+1, instance) - elif instance in self._images: - index = self._images.index(instance) - self._images.insert(index, instance.feature_provider) + if instance not in self._images: + if instance.feature_provider: + if instance.feature_provider in self._images: + index = self._images.index(instance.feature_provider) + self._images.insert(index+1, instance) + else: + self._images.append(instance.feature_provider) + self._images.append(instance) + else: + self._images.append(instance) else: - self._images.append(instance.feature_provider) - self._images.append(instance) - + if instance.feature_provider: + if instance.feature_provider not in self._images: + index = self._images.index(instance) + self._images.insert(index, instance.feature_provider) def _generate_build_script(self): build_commands = [] + obselete_images = [] for each_instance in self._images: if isinstance(each_instance.definition.implementation, DockerFile): tag = self._docker_tag_for(each_instance) + obselete_images.append(tag) folder = "./" + each_instance.name command = self.BUILD_COMMAND.format(folder=folder, tag=tag) build_commands.append(command) - build_script = self._build_script() - with open(build_script, "w") as stream: - content = self.BUILD_SCRIPT_TEXT.format("\n".join(build_commands)) - stream.write(content) + script = self._build_script() + with open(script, "w") as stream: + body = self._fetch_script_template() + body = sub(self.BUILD_COMMAND_MARKER, + "\n\t".join(build_commands), + body) + body = sub(self.OBSELETE_IMAGES_MARKER, + " ".join(obselete_images), + body) + stream.write(body) + BUILD_COMMAND_MARKER = "{build_commands}" - BUILD_COMMAND = "docker build --no-cache -t {tag} {folder}" + OBSELETE_IMAGES_MARKER = "{obselete_images}" - def _build_script(self): - return join_paths(self._image_directory, "build_images.sh") + def _fetch_script_template(self): + return get_data('camp', 'data/manage_images.sh').decode("utf-8") - BUILD_SCRIPT_TEXT = ("#!/bin/bash\n" - "#\n" - "# Generated by CAMP. Edit carefully\n" - "#\n" - "# Build all images and set the appropriate tags\n" - "#\n" - "set -e\n" - "{0}\n" - "echo 'All images ready.'\n") + # Issue 82: The "--force-rm" option avoids generating many + # dangling images and consuming a lot of disk space + BUILD_COMMAND = "docker build --force-rm --no-cache -t {tag} {folder}" + + def _build_script(self): + return join_paths(self._image_directory, "build_images.sh") diff --git a/samples/java/template/greetings/Dockerfile b/samples/java/template/greetings/Dockerfile index 452c0b1c..78244d62 100644 --- a/samples/java/template/greetings/Dockerfile +++ b/samples/java/template/greetings/Dockerfile @@ -11,7 +11,8 @@ # Step 1: Build the WAR file FROM openjdk:8-jdk-stretch as builder -LABEL maintainer "franck.chauvel@sintef.no" +LABEL maintainer="franck.chauvel@sintef.no" +LABEL stage="intermediate" RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/samples/stamp/ow2/template/lutece/Dockerfile b/samples/stamp/ow2/template/lutece/Dockerfile index efc7f404..6f203468 100644 --- a/samples/stamp/ow2/template/lutece/Dockerfile +++ b/samples/stamp/ow2/template/lutece/Dockerfile @@ -13,6 +13,8 @@ ARG site=site-forms-demo-1.0.0-SNAPSHOT # A first container to build the lutece web app FROM debian:stretch as builder +LABEL stage="intermediate" + RUN apt-get update && apt-get dist-upgrade -y && \ apt-get install -y --no-install-recommends \ mysql-client \ diff --git a/setup.py b/setup.py index 5ba38c45..55ce2f32 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,10 @@ packages=find_packages(exclude=["tests*", "tests.*"]), include_package_data = True, package_data = { - "camp": ["data/metamodel.yml"] + "camp": [ + "data/metamodel.yml", + "data/manage_images.sh" + ] }, license="MIT", test_suite="tests", diff --git a/tests/realize/test_docker.py b/tests/realize/test_docker.py index 279374a4..ffddd0d2 100644 --- a/tests/realize/test_docker.py +++ b/tests/realize/test_docker.py @@ -117,9 +117,9 @@ def test_when_the_stack_has_more_than_two_components(self): self.build(configuration) expected_command_order = ( - "docker build --no-cache -t camp-jdk_0 ./jdk_0\n" - "docker build --no-cache -t camp-tomcat_0 ./tomcat_0\n" - "docker build --no-cache -t camp-server_0 ./server_0\n" + "docker build --force-rm --no-cache -t camp-jdk_0 ./jdk_0\n" + "\tdocker build --force-rm --no-cache -t camp-tomcat_0 ./tomcat_0\n" + "\tdocker build --force-rm --no-cache -t camp-server_0 ./server_0\n" ) self.assert_generated( diff --git a/tests/realize/test_operators.py b/tests/realize/test_operators.py index 2cbe16a8..84544de2 100644 --- a/tests/realize/test_operators.py +++ b/tests/realize/test_operators.py @@ -206,5 +206,5 @@ def test_select_resource_before_substitutions_take_place(self): self.assertIn("nginx_variable=something_else", each_configuration.content_of("docker-compose.yml")) - self.assertIn("build: ./images/nginx_0", + self.assertIn("image: camp-nginx_0", each_configuration.content_of("docker-compose.yml"))