diff --git a/py/MANIFEST.in b/py/MANIFEST.in index ce3c6e809..716a0e14a 100644 --- a/py/MANIFEST.in +++ b/py/MANIFEST.in @@ -4,4 +4,3 @@ include README.md include google/fhir/py.typed include proto/google/fhir/proto/py.typed recursive-include proto/google/fhir/proto *.pyi -global-exclude *_test.py \ No newline at end of file diff --git a/py/distribution/Dockerfile b/py/distribution/Dockerfile new file mode 100644 index 000000000..57b8253ee --- /dev/null +++ b/py/distribution/Dockerfile @@ -0,0 +1,25 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.8.6-buster + +# Copy repository contents and setup working directory +COPY . fhir +WORKDIR /fhir/py + +# Establish host <=> container volume for release artifacts +RUN mkdir -p /tmp/google/fhir/release +VOLUME /tmp/google/fhir/release + +ENTRYPOINT [ "distribution/build_distribution.sh" ] diff --git a/py/distribution/README.md b/py/distribution/README.md new file mode 100644 index 000000000..03664dce4 --- /dev/null +++ b/py/distribution/README.md @@ -0,0 +1,90 @@ +# Overview + +This directory contains scripts to build binary and source distributions of the +`google-fhir` Python package. + +## Building and testing the release + +To generate `sdist` and `bdist_wheel` release artifacts, it is required that the +host operating system be a supported Linux distribution (Debian >= buster or +equivalent Ubuntu distro). While only _required_ for Darwin hosts, the usage of +Docker is encouraged to obtain the most hermetically-selead environment when +building and testing. + +See the [official documentation](https://docs.docker.com/get-docker/) for more +on installing Docker for your host. + +### Linux + +Run `./distribution/build_distribution.sh` from the `//fhir/py/` directory: + +``` +un@host:/tmp/fhir/py$ ./distribution/build_distribution.sh +``` + +### Docker + +The Docker image must be built from the **project root** (`//fhir/`), since we +include file outside of the immediate Docker [build context](https://docs.docker.com/engine/reference/commandline/build/#extended-description). + +First, build the image: + +``` +un@host:/tmp/fhir$ docker build -f py/distribution/Dockerfile . +``` + +Next, create a directory where you want the resulting artifacts to be placed: + +``` +un@host:/tmp/fhir$ mkdir -p +``` + +Finally, run the container, and mount your ``: + +``` +un@host:/tmp/fhir$ docker run -it --rm -v :/tmp/google/fhir/release +``` + +Where: + +* `-it`: Tells Docker that we want to run the container interactvely, by keeping +`STDIN` open even if not attached. It additional allocates a pseudo-TTY +* `--rm`: Instructs Docker to automatically remove the container once it exists +* `-v`: Mounts the host directory: `` at `/tmp/google/fhir/release`, +where the resulting artifacts are generated inside the container + +### General + +The `build_distribution.sh` will carry out the following steps: + +1. Install necessary system dependencies (at time of writing, `protoc`) +2. Install necessary Python dependencies within a virtualenv +3. Build `sdist` and `bdist_wheel` Python artifacts +4. Instantiate a sandbox "workspace" with appropriate references to necessary + runtime testdata +5. Execute all tests against both `sdist` and `bdist_wheel` distributions + +If all tests pass, the `sdist` and `bdist_wheel` release artifacts are placed in +`/tmp/google/fhir/release`. + +## Installing the release + +The output generated from `build_distribution.sh` can be directly installed. + +To install the `sdist`, simply: + +``` +un@host:/tmp/fhir$ pip3 install /*.tar.gz +``` + +Similarly, to install the `bdist_wheel`: + +``` +un@host:/tmp/fhir$ pip3 install /*.whl +``` + +It is recommend that these steps be performed in a Python virtual environment +such as [virtualenv](https://pypi.org/project/virtualenv/) or [venv](https://docs.python.org/3/library/venv.html). + +See more about Python packaging from the [official Python packaging +documentation](https://packaging.python.org/tutorials/packaging-projects/). diff --git a/py/distribution/build_distribution.sh b/py/distribution/build_distribution.sh new file mode 100755 index 000000000..3ff345a05 --- /dev/null +++ b/py/distribution/build_distribution.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script creates an sdist and bdist_wheel of google-fhir for Debian/Darwin +# platforms. Both sdist and bdist_wheel artifacts are installed into a local +# virtualenv and tested against the full suite of tests (although the test data +# is not actually shipped with the artifacts themselves). +# +# This script should be executed from //py/ and, while not required, is best if +# executed inside a Docker container. See README.md for more information. + +set -e +set -E +set -u + +FHIR_ROOT="${PWD}/.." +FHIR_WORKSPACE='com_google_fhir' +PROTOBUF_URL='https://github.com/protocolbuffers/protobuf/releases/download' +PROTOC_SHA='4a3b26d1ebb9c1d23e933694a6669295f6a39ddc64c3db2adf671f0a6026f82e' +PROTOC_VERSION='3.13.0' + +# Helper around print statements. +function print_info() { + + function print_sep() { + echo "$(seq -s'=' 0 "$(( $(tput cols) - 1))" | tr -d '[:digit:]')" + } + + print_sep + echo "$1" + print_sep +} + +# Download and install prebuilt Linux binary from URL. URL should point to a +# compressed (.zip) resource. +# +# Globals: +# None +# Arguments: +# url: The URL to fetch. +# resource: The resource at the basename of the URL (a .zip file). +# sha: The SHA256 hash of the resource for verification. +# destination: The destination directory to install to. The user must have +# sufficient read/write privileges or an error will be raised. +function install_prebuilt_binary() { + pushd /tmp + + local -r url="$1" + local -r resource="$2" + local -r sha="$3" + local -r destination="$4" + + if [[ -f "${destination}" ]]; then + echo "File exists at: ${destination}." + exit 1 + fi + + print_info "Installing ${resource}..." + curl -OL "${url}" + echo "${sha} ${resource}" | sha256sum --check + unzip -o "${resource}" -d "${destination}" + + popd +} + +# Helper method to execute all test scripts. The google-fhir package should be +# installed in an active virtual environment prior to calling. +function test_google_fhir() { + local -r workspace="$1" + pushd "${workspace}" + find -L . -type f -name '*_test.py' -not -path '*/build/*' -print0 | \ + xargs -0 -n1 python3 -I + popd +} + +# Sets up a google-fhir "workspace" with appropriate subdirectory structure for +# executing the full test suite against an installed google-fhir package. +# +# Globals: +# FHIR_ROOT: The parent directory of //py/; necessary for testdata/ and spec/. +# FHIR_WORKSPACE: The Bazel workspace identifier of google-fhir. +# Arguments: +# workspace: An ephemeral directory for staging+testing distributions. +function initialize_workspace() { + local -r workspace="$1" + mkdir -p "${workspace}/${FHIR_WORKSPACE}" + ln -s "${FHIR_ROOT}/testdata" "${workspace}/${FHIR_WORKSPACE}/testdata" + ln -s "${FHIR_ROOT}/spec" "${workspace}/${FHIR_WORKSPACE}/spec" + ln -s "${FHIR_ROOT}/py" "${workspace}/py" +} + +# Removes the ephemeral workspace and pypi artifacts. +# +# Globals: +# None +# Arguments: +# workspace: An ephemeral directory for staging+testing distributions. +function cleanup() { + local -r workspace="$1" + rm -rf "${workspace}" && \ + rm -rf *.egg-info && \ + rm -rf build && \ + rm -rf dist +} + +function main() { + # TODO: Should try and perform some version validation to be more strict + if [[ "$(command -v protoc &> /dev/null; echo $?)" -ne 0 ]]; then + local -r protoc_resource="protoc-${PROTOC_VERSION}-linux-x86_64.zip" + local -r protoc_url="${PROTOBUF_URL}/v${PROTOC_VERSION}/${protoc_resource}" + install_prebuilt_binary "${protoc_url}" \ + "${protoc_resource}" \ + "${PROTOC_SHA}" \ + '/usr/local' + fi + + # Set the Python version and standup a virtualenv + print_info 'Instantiating Python virtualenv...' + pip3 install virtualenv + python3 -m virtualenv /tmp/venv + source /tmp/venv/bin/activate + + # Install/upgrade necessary distutils + pip3 install --upgrade pip + pip3 install --upgrade setuptools + pip3 install wheel + + # Generate output into a "release" subdir + # TODO: Separate "build" reqs (e.g. mypy-protobuf) from "install" reqs + print_info 'Building distribution...' + pip3 install -r requirements.txt + python3 setup.py sdist bdist_wheel + + local -r workspace="$(mktemp -d -t fhir-XXXXXXXXXX)" + print_info "Initializing workspace ${workspace}..." + initialize_workspace "${workspace}" + trap "cleanup ${workspace}" EXIT + + print_info 'Testing sdist...' + pip3 install dist/*.tar.gz && test_google_fhir "${workspace}" && \ + pip3 uninstall -y google-fhir + + print_info 'Testing bdist_wheel...' + pip3 install dist/*.whl && test_google_fhir "${workspace}" && \ + pip3 uninstall -y google-fhir + + + print_info 'Staging artifacts at /tmp/google/fhir/release/...' + mkdir -p /tmp/google/fhir/release + cp dist/* /tmp/google/fhir/release/ +} + +main "$@" diff --git a/py/google/fhir/r4/extensions_test.py b/py/google/fhir/r4/extensions_test.py index 6a5e1f420..7cb5206b6 100644 --- a/py/google/fhir/r4/extensions_test.py +++ b/py/google/fhir/r4/extensions_test.py @@ -15,6 +15,7 @@ """Test extensions functionality.""" import os +import sys from typing import Type from google.protobuf import message @@ -26,7 +27,12 @@ from proto.google.fhir.proto.r4.core.resources import patient_pb2 from google.fhir import extensions from google.fhir import extensions_test -from testdata.r4.profiles import test_extensions_pb2 + +try: + from testdata.r4.profiles import test_extensions_pb2 +except ImportError: + # TODO: Add test protos to PYTHONPATH during dist testing. + pass # Fall through _EXTENSIONS_DIR = os.path.join('testdata', 'r4', 'extensions') @@ -125,10 +131,16 @@ def testMessageToExtension_withCapabilityStatementSearchParameterCombination_suc 'capability', extensions_pb2.CapabilityStatementSearchParameterCombination) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testExtensionToMessage_withDigitalMediaType_succeeds(self): self.assert_extension_to_message_equals_golden( 'digital_media_type', test_extensions_pb2.DigitalMediaType) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testMessageToExtension_withDigitalMediaType_succeeds(self): self.assert_message_to_extension_equals_golden( 'digital_media_type', test_extensions_pb2.DigitalMediaType) diff --git a/py/google/fhir/stu3/extensions_test.py b/py/google/fhir/stu3/extensions_test.py index 190a69eea..379e326d6 100644 --- a/py/google/fhir/stu3/extensions_test.py +++ b/py/google/fhir/stu3/extensions_test.py @@ -15,6 +15,7 @@ """Test extensions functionality.""" import os +import sys from typing import Type from google.protobuf import message @@ -26,7 +27,13 @@ from proto.google.fhir.proto.stu3 import resources_pb2 from google.fhir import extensions from google.fhir import extensions_test -from testdata.stu3.profiles import test_extensions_pb2 + +try: + from testdata.stu3.profiles import test_extensions_pb2 +except ImportError: + # TODO: Add test protos to PYTHONPATH during dist testing. + pass # Fall through + _EXTENSIONS_DIR = os.path.join('testdata', 'stu3', 'extensions') @@ -125,10 +132,16 @@ def testMessageToExtension_withCapabilityStatementSearchParameterCombination_suc 'capability', extensions_pb2.CapabilityStatementSearchParameterCombination) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testExtensionToMessage_withDigitalMediaType_succeeds(self): self.assert_extension_to_message_equals_golden( 'digital_media_type', test_extensions_pb2.DigitalMediaType) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testMessageToExtension_withDigitalMediaType_succeeds(self): self.assert_message_to_extension_equals_golden( 'digital_media_type', test_extensions_pb2.DigitalMediaType) diff --git a/py/google/fhir/utils/annotation_utils_test.py b/py/google/fhir/utils/annotation_utils_test.py index 9c63e6265..a2f4a02b0 100644 --- a/py/google/fhir/utils/annotation_utils_test.py +++ b/py/google/fhir/utils/annotation_utils_test.py @@ -14,6 +14,8 @@ # limitations under the License. """Test annotation_utils functionality.""" +import sys + from google.protobuf import descriptor_pb2 from absl.testing import absltest from proto.google.fhir.proto.r4 import uscore_codes_pb2 @@ -23,7 +25,12 @@ from proto.google.fhir.proto.r4.core.resources import observation_pb2 from proto.google.fhir.proto.r4.core.resources import patient_pb2 from google.fhir.utils import annotation_utils -from testdata.r4.profiles import test_pb2 + +try: + from testdata.r4.profiles import test_pb2 +except ImportError: + # TODO: Add test protos to PYTHONPATH during dist testing. + pass # Fall through _ADDRESS_USECODE_FHIR_VALUESET_URL = 'http://hl7.org/fhir/ValueSet/address-use' _BODY_LENGTH_UNITS_VALUESET_URL = 'http://hl7.org/fhir/ValueSet/ucum-bodylength' @@ -143,6 +150,9 @@ def testIsReference_withInvalidReferenceType_returnsFalse(self): self.assertFalse(annotation_utils.is_reference(boolean_descriptor_proto)) self.assertFalse(annotation_utils.is_reference(code_descriptor_proto)) + @absltest.skipIf( + 'testdata' not in sys.modules, + 'google-fhir package does not build+install tertiary testdata protos.') def testGetFixedCodingSystem_withValidFixedCodingSystem_returnsValue(self): """Test get_fixed_coding_system functionality when annotation is present.""" expected_system = 'http://hl7.org/fhir/metric-color' diff --git a/py/google/fhir/utils/proto_utils_test.py b/py/google/fhir/utils/proto_utils_test.py index 40becd491..dde964172 100644 --- a/py/google/fhir/utils/proto_utils_test.py +++ b/py/google/fhir/utils/proto_utils_test.py @@ -306,9 +306,11 @@ def testCopyCommonField_notPresentInBothMessages_raisesException(self): def testGetMessageClassFromDescriptor_returnsMessageClass(self): """Tests that the correct class is returned for a message.""" - self.assertEqual( - proto_utils.get_message_class_from_descriptor( - patient_pb2.Patient.DESCRIPTOR), patient_pb2.Patient) + actual = proto_utils.get_message_class_from_descriptor( + patient_pb2.Patient.DESCRIPTOR) + self.assertTrue( + proto_utils.are_same_message_type(actual.DESCRIPTOR, + patient_pb2.Patient.DESCRIPTOR)) def testCreateMessageFromDescriptor_returnsMessage(self): """Tests that the correct class is returned for a message."""