From d98ce49672cfa6c978889f987ae1115b9208c9c3 Mon Sep 17 00:00:00 2001 From: Cameron Tew Date: Thu, 19 Nov 2020 10:16:45 -0800 Subject: [PATCH] Add .sh script and Dockerfile for building and testing distribution artifacts. Distribution artifacts can be built locally on Linux host systems or otherwise can be built using Docker (the latter is encouraged for a more hermetically-sealed environment). PiperOrigin-RevId: 343316668 --- py/MANIFEST.in | 1 - py/distribution/Dockerfile | 25 +++ py/distribution/README.md | 90 ++++++++++ py/distribution/build_distribution.sh | 165 ++++++++++++++++++ py/google/fhir/r4/extensions_test.py | 14 +- py/google/fhir/stu3/extensions_test.py | 15 +- py/google/fhir/utils/annotation_utils_test.py | 12 +- py/google/fhir/utils/proto_utils_test.py | 8 +- 8 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 py/distribution/Dockerfile create mode 100644 py/distribution/README.md create mode 100755 py/distribution/build_distribution.sh diff --git a/py/MANIFEST.in b/py/MANIFEST.in index ce3c6e809..716a0e14a 100644 --- a/py/MANIFEST.in +++ b/py/MANIFEST.in @@ -4,4 +4,3 @@ include README.md include google/fhir/py.typed include proto/google/fhir/proto/py.typed recursive-include proto/google/fhir/proto *.pyi -global-exclude *_test.py \ No newline at end of file diff --git a/py/distribution/Dockerfile b/py/distribution/Dockerfile new file mode 100644 index 000000000..57b8253ee --- /dev/null +++ b/py/distribution/Dockerfile @@ -0,0 +1,25 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.8.6-buster + +# Copy repository contents and setup working directory +COPY . fhir +WORKDIR /fhir/py + +# Establish host <=> container volume for release artifacts +RUN mkdir -p /tmp/google/fhir/release +VOLUME /tmp/google/fhir/release + +ENTRYPOINT [ "distribution/build_distribution.sh" ] diff --git a/py/distribution/README.md b/py/distribution/README.md new file mode 100644 index 000000000..03664dce4 --- /dev/null +++ b/py/distribution/README.md @@ -0,0 +1,90 @@ +# Overview + +This directory contains scripts to build binary and source distributions of the +`google-fhir` Python package. + +## Building and testing the release + +To generate `sdist` and `bdist_wheel` release artifacts, it is required that the +host operating system be a supported Linux distribution (Debian >= buster or +equivalent Ubuntu distro). While only _required_ for Darwin hosts, the usage of +Docker is encouraged to obtain the most hermetically-selead environment when +building and testing. + +See the [official documentation](https://docs.docker.com/get-docker/) for more +on installing Docker for your host. + +### Linux + +Run `./distribution/build_distribution.sh` from the `//fhir/py/` directory: + +``` +un@host:/tmp/fhir/py$ ./distribution/build_distribution.sh +``` + +### Docker + +The Docker image must be built from the **project root** (`//fhir/`), since we +include file outside of the immediate Docker [build context](https://docs.docker.com/engine/reference/commandline/build/#extended-description). + +First, build the image: + +``` +un@host:/tmp/fhir$ docker build -f py/distribution/Dockerfile . +``` + +Next, create a directory where you want the resulting artifacts to be placed: + +``` +un@host:/tmp/fhir$ mkdir -p +``` + +Finally, run the container, and mount your ``: + +``` +un@host:/tmp/fhir$ docker run -it --rm -v :/tmp/google/fhir/release +``` + +Where: + +* `-it`: Tells Docker that we want to run the container interactvely, by keeping +`STDIN` open even if not attached. It additional allocates a pseudo-TTY +* `--rm`: Instructs Docker to automatically remove the container once it exists +* `-v`: Mounts the host directory: `` at `/tmp/google/fhir/release`, +where the resulting artifacts are generated inside the container + +### General + +The `build_distribution.sh` will carry out the following steps: + +1. Install necessary system dependencies (at time of writing, `protoc`) +2. Install necessary Python dependencies within a virtualenv +3. Build `sdist` and `bdist_wheel` Python artifacts +4. Instantiate a sandbox "workspace" with appropriate references to necessary + runtime testdata +5. Execute all tests against both `sdist` and `bdist_wheel` distributions + +If all tests pass, the `sdist` and `bdist_wheel` release artifacts are placed in +`/tmp/google/fhir/release`. + +## Installing the release + +The output generated from `build_distribution.sh` can be directly installed. + +To install the `sdist`, simply: + +``` +un@host:/tmp/fhir$ pip3 install /*.tar.gz +``` + +Similarly, to install the `bdist_wheel`: + +``` +un@host:/tmp/fhir$ pip3 install /*.whl +``` + +It is recommend that these steps be performed in a Python virtual environment +such as [virtualenv](https://pypi.org/project/virtualenv/) or [venv](https://docs.python.org/3/library/venv.html). + +See more about Python packaging from the [official Python packaging +documentation](https://packaging.python.org/tutorials/packaging-projects/). diff --git a/py/distribution/build_distribution.sh b/py/distribution/build_distribution.sh new file mode 100755 index 000000000..3ff345a05 --- /dev/null +++ b/py/distribution/build_distribution.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script creates an sdist and bdist_wheel of google-fhir for Debian/Darwin +# platforms. Both sdist and bdist_wheel artifacts are installed into a local +# virtualenv and tested against the full suite of tests (although the test data +# is not actually shipped with the artifacts themselves). +# +# This script should be executed from //py/ and, while not required, is best if +# executed inside a Docker container. See README.md for more information. + +set -e +set -E +set -u + +FHIR_ROOT="${PWD}/.." +FHIR_WORKSPACE='com_google_fhir' +PROTOBUF_URL='https://github.com/protocolbuffers/protobuf/releases/download' +PROTOC_SHA='4a3b26d1ebb9c1d23e933694a6669295f6a39ddc64c3db2adf671f0a6026f82e' +PROTOC_VERSION='3.13.0' + +# Helper around print statements. +function print_info() { + + function print_sep() { + echo "$(seq -s'=' 0 "$(( $(tput cols) - 1))" | tr -d '[:digit:]')" + } + + print_sep + echo "$1" + print_sep +} + +# Download and install prebuilt Linux binary from URL. URL should point to a +# compressed (.zip) resource. +# +# Globals: +# None +# Arguments: +# url: The URL to fetch. +# resource: The resource at the basename of the URL (a .zip file). +# sha: The SHA256 hash of the resource for verification. +# destination: The destination directory to install to. The user must have +# sufficient read/write privileges or an error will be raised. +function install_prebuilt_binary() { + pushd /tmp + + local -r url="$1" + local -r resource="$2" + local -r sha="$3" + local -r destination="$4" + + if [[ -f "${destination}" ]]; then + echo "File exists at: ${destination}." + exit 1 + fi + + print_info "Installing ${resource}..." + curl -OL "${url}" + echo "${sha} ${resource}" | sha256sum --check + unzip -o "${resource}" -d "${destination}" + + popd +} + +# Helper method to execute all test scripts. The google-fhir package should be +# installed in an active virtual environment prior to calling. +function test_google_fhir() { + local -r workspace="$1" + pushd "${workspace}" + find -L . -type f -name '*_test.py' -not -path '*/build/*' -print0 | \ + xargs -0 -n1 python3 -I + popd +} + +# Sets up a google-fhir "workspace" with appropriate subdirectory structure for +# executing the full test suite against an installed google-fhir package. +# +# Globals: +# FHIR_ROOT: The parent directory of //py/; necessary for testdata/ and spec/. +# FHIR_WORKSPACE: The Bazel workspace identifier of google-fhir. +# Arguments: +# workspace: An ephemeral directory for staging+testing distributions. +function initialize_workspace() { + local -r workspace="$1" + mkdir -p "${workspace}/${FHIR_WORKSPACE}" + ln -s "${FHIR_ROOT}/testdata" "${workspace}/${FHIR_WORKSPACE}/testdata" + ln -s "${FHIR_ROOT}/spec" "${workspace}/${FHIR_WORKSPACE}/spec" + ln -s "${FHIR_ROOT}/py" "${workspace}/py" +} + +# Removes the ephemeral workspace and pypi artifacts. +# +# Globals: +# None +# Arguments: +# workspace: An ephemeral directory for staging+testing distributions. +function cleanup() { + local -r workspace="$1" + rm -rf "${workspace}" && \ + rm -rf *.egg-info && \ + rm -rf build && \ + rm -rf dist +} + +function main() { + # TODO: Should try and perform some version validation to be more strict + if [[ "$(command -v protoc &> /dev/null; echo $?)" -ne 0 ]]; then + local -r protoc_resource="protoc-${PROTOC_VERSION}-linux-x86_64.zip" + local -r protoc_url="${PROTOBUF_URL}/v${PROTOC_VERSION}/${protoc_resource}" + install_prebuilt_binary "${protoc_url}" \ + "${protoc_resource}" \ + "${PROTOC_SHA}" \ + '/usr/local' + fi + + # Set the Python version and standup a virtualenv + print_info 'Instantiating Python virtualenv...' + pip3 install virtualenv + python3 -m virtualenv /tmp/venv + source /tmp/venv/bin/activate + + # Install/upgrade necessary distutils + pip3 install --upgrade pip + pip3 install --upgrade setuptools + pip3 install wheel + + # Generate output into a "release" subdir + # TODO: Separate "build" reqs (e.g. mypy-protobuf) from "install" reqs + print_info 'Building distribution...' + pip3 install -r requirements.txt + python3 setup.py sdist bdist_wheel + + local -r workspace="$(mktemp -d -t fhir-XXXXXXXXXX)" + print_info "Initializing workspace ${workspace}..." + initialize_workspace "${workspace}" + trap "cleanup ${workspace}" EXIT + + print_info 'Testing sdist...' + pip3 install dist/*.tar.gz && test_google_fhir "${workspace}" && \ + pip3 uninstall -y google-fhir + + print_info 'Testing bdist_wheel...' + pip3 install dist/*.whl && test_google_fhir "${workspace}" && \ + pip3 uninstall -y google-fhir + + + print_info 'Staging artifacts at /tmp/google/fhir/release/...' + mkdir -p /tmp/google/fhir/release + cp dist/* /tmp/google/fhir/release/ +} + +main "$@" diff --git a/py/google/fhir/r4/extensions_test.py b/py/google/fhir/r4/extensions_test.py index 6a5e1f420..7cb5206b6 100644 --- a/py/google/fhir/r4/extensions_test.py +++ b/py/google/fhir/r4/extensions_test.py @@ -15,6 +15,7 @@ """Test extensions functionality.""" import os +import sys from typing import Type from google.protobuf import message @@ -26,7 +27,12 @@ from proto.google.fhir.proto.r4.core.resources import patient_pb2 from google.fhir import extensions from google.fhir import extensions_test -from testdata.r4.profiles import test_extensions_pb2 + +try: + from testdata.r4.profiles import test_extensions_pb2 +except ImportError: + # TODO: Add test protos to PYTHONPATH during dist testing. + pass # Fall through _EXTENSIONS_DIR = os.path.join('testdata', 'r4', 'extensions') @@ -125,10 +131,16 @@ def testMessageToExtension_withCapabilityStatementSearchParameterCombination_suc 'capability', extensions_pb2.CapabilityStatementSearchParameterCombination) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testExtensionToMessage_withDigitalMediaType_succeeds(self): self.assert_extension_to_message_equals_golden( 'digital_media_type', test_extensions_pb2.DigitalMediaType) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testMessageToExtension_withDigitalMediaType_succeeds(self): self.assert_message_to_extension_equals_golden( 'digital_media_type', test_extensions_pb2.DigitalMediaType) diff --git a/py/google/fhir/stu3/extensions_test.py b/py/google/fhir/stu3/extensions_test.py index 190a69eea..379e326d6 100644 --- a/py/google/fhir/stu3/extensions_test.py +++ b/py/google/fhir/stu3/extensions_test.py @@ -15,6 +15,7 @@ """Test extensions functionality.""" import os +import sys from typing import Type from google.protobuf import message @@ -26,7 +27,13 @@ from proto.google.fhir.proto.stu3 import resources_pb2 from google.fhir import extensions from google.fhir import extensions_test -from testdata.stu3.profiles import test_extensions_pb2 + +try: + from testdata.stu3.profiles import test_extensions_pb2 +except ImportError: + # TODO: Add test protos to PYTHONPATH during dist testing. + pass # Fall through + _EXTENSIONS_DIR = os.path.join('testdata', 'stu3', 'extensions') @@ -125,10 +132,16 @@ def testMessageToExtension_withCapabilityStatementSearchParameterCombination_suc 'capability', extensions_pb2.CapabilityStatementSearchParameterCombination) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testExtensionToMessage_withDigitalMediaType_succeeds(self): self.assert_extension_to_message_equals_golden( 'digital_media_type', test_extensions_pb2.DigitalMediaType) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testMessageToExtension_withDigitalMediaType_succeeds(self): self.assert_message_to_extension_equals_golden( 'digital_media_type', test_extensions_pb2.DigitalMediaType) diff --git a/py/google/fhir/utils/annotation_utils_test.py b/py/google/fhir/utils/annotation_utils_test.py index 9c63e6265..a2f4a02b0 100644 --- a/py/google/fhir/utils/annotation_utils_test.py +++ b/py/google/fhir/utils/annotation_utils_test.py @@ -14,6 +14,8 @@ # limitations under the License. """Test annotation_utils functionality.""" +import sys + from google.protobuf import descriptor_pb2 from absl.testing import absltest from proto.google.fhir.proto.r4 import uscore_codes_pb2 @@ -23,7 +25,12 @@ from proto.google.fhir.proto.r4.core.resources import observation_pb2 from proto.google.fhir.proto.r4.core.resources import patient_pb2 from google.fhir.utils import annotation_utils -from testdata.r4.profiles import test_pb2 + +try: + from testdata.r4.profiles import test_pb2 +except ImportError: + # TODO: Add test protos to PYTHONPATH during dist testing. + pass # Fall through _ADDRESS_USECODE_FHIR_VALUESET_URL = 'http://hl7.org/fhir/ValueSet/address-use' _BODY_LENGTH_UNITS_VALUESET_URL = 'http://hl7.org/fhir/ValueSet/ucum-bodylength' @@ -143,6 +150,9 @@ def testIsReference_withInvalidReferenceType_returnsFalse(self): self.assertFalse(annotation_utils.is_reference(boolean_descriptor_proto)) self.assertFalse(annotation_utils.is_reference(code_descriptor_proto)) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testGetFixedCodingSystem_withValidFixedCodingSystem_returnsValue(self): """Test get_fixed_coding_system functionality when annotation is present.""" expected_system = 'http://hl7.org/fhir/metric-color' diff --git a/py/google/fhir/utils/proto_utils_test.py b/py/google/fhir/utils/proto_utils_test.py index 40becd491..dde964172 100644 --- a/py/google/fhir/utils/proto_utils_test.py +++ b/py/google/fhir/utils/proto_utils_test.py @@ -306,9 +306,11 @@ def testCopyCommonField_notPresentInBothMessages_raisesException(self): def testGetMessageClassFromDescriptor_returnsMessageClass(self): """Tests that the correct class is returned for a message.""" - self.assertEqual( - proto_utils.get_message_class_from_descriptor( - patient_pb2.Patient.DESCRIPTOR), patient_pb2.Patient) + actual = proto_utils.get_message_class_from_descriptor( + patient_pb2.Patient.DESCRIPTOR) + self.assertTrue( + proto_utils.are_same_message_type(actual.DESCRIPTOR, + patient_pb2.Patient.DESCRIPTOR)) def testCreateMessageFromDescriptor_returnsMessage(self): """Tests that the correct class is returned for a message."""