Merge pull request #113 from EliLillyCo/release/1.3.0

Release/1.3.0

.travis.yml

@@ -14,12 +14,11 @@ cache:
 python:
 - '3.6'
 - '3.7'
-#- '3.8'
+- '3.8'
 install:
 - pip install --upgrade pip wheel
-- pip install -r requirements.txt
+- make install_development_requirements
 - make install
-- make install_extras
 before_script:
 - wget https://github.com/broadinstitute/cromwell/releases/download/45/cromwell-45.jar -O /tmp/cromwell-45.jar
 - export CROMWELL_JAR=/tmp/cromwell-45.jar

CHANGES.md

@@ -1,5 +1,14 @@
 # Changes
 
+## v1.3.0 (2020.02.14)
+
+* Added support for testing workflows on DNAnexus using [dxWDL](https://github.com/dnanexus/dxWDL)
+* Using DNAnexus-related features will cause the user to be automatically logged in to their account if not already logged in (log-in is interactive unless account credentials or a token are specified in the config file)
+* Enhanced support for [miniwdl]() and made it the default executor (to configure different default executor(s), see the [manual](https://pytest-wdl.readthedocs.io/en/stable/manual.html#configuration))
+* Generate requirements.txt using `pip freeze`. Note that miniwdl is pegged to a specific version (0.6.4) because we depend on internal functionality; this limitation will be fixed in a future release
+* Many improvements to the manual (thanks @mlin for suggestions), including a new "quickstart" example project
+* Bugfix: within-project local data file resolution was implemented incorrectly for the case where there are packages (i.e. `__init__.py` files) in the `tests/` folder - it is now fixed.
+
 ## v1.2.1 (2019.12.12)
 
 * Fixes #93 - ensures parent directories exist when localizing a file

Makefile

@@ -4,7 +4,7 @@ tests = tests
 # Use this option to show full stack trace for errors
 #pytestopts = --full-trace
 #pytestopts = -ra --tb=short
-pytestopts = -s -vv --show-capture=all
+pytestopts =  -s -vv --show-capture=all -m "not remote"
 #pytestopts = -s -vv --show-capture=all -m "not integration"
 
 all: clean install install_extras install_development_requirements test test_release_setup
@@ -15,6 +15,7 @@ install: clean
 
 install_development_requirements:
 	pip install -r requirements.txt
+	pip install -r requirements-dev.txt
 
 install_extras:
 	pip install .[all]
@@ -36,6 +37,7 @@ reformat:
 
 clean:
 	rm -f .coverage
+	rm -f coverage.xml
 	rm -Rf .eggs
 	rm -Rf .pytest_cache
 	rm -Rf __pycache__
@@ -45,6 +47,7 @@ clean:
 	rm -Rf **/*.pyc
 	rm -Rf dist/
 	rm -Rf build/
+	rm -Rf docs/build
 	rm -Rf $(package).egg-info
 	rm -Rf cromwell-workflow-logs
 

README.md

@@ -8,10 +8,13 @@ This package is a plugin for the [pytest](https://docs.pytest.org/en/latest/) un
 
 ## Dependencies
 
-* Python 3.6 or 3.7 (3.8 is not yet fully supported)
-* Java 1.8+
-* [Cromwell](https://github.com/broadinstitute/cromwell/releases/tag/38) JAR file
-* [Docker](https://www.docker.com/get-started) daemon (if your WDL tasks depend on Docker images)
+* Python 3.6+
+* At least one of the supported workflow engines:
+    * [Miniwdl](https://github.com/chanzuckerberg/miniwdl) - automatically installed as a dependency of pytest-wdl
+    * [Cromwell](https://github.com/broadinstitute/cromwell/releases/tag/38) JAR file
+    * [dxWDL](https://github.com/dnanexus/dxWDL) JAR file
+* Java-based workflow engines (e.g. Cromwell and dxWDL) require a Java runtime (typically 1.8+)
+* If your WDL tasks depend on Docker images, make sure to have the [Docker](https://www.docker.com/get-started) daemon running
 
 Other python dependencies are installed when you install the library.
 
@@ -39,21 +42,18 @@ $ pip install git+https://github.com/elilillyco/pytest-wdl.git
 
 ### Install optional dependencies
 
-Some optional features of pytest-wdl have additional dependencies that are loaded on-demand. For example, to enable comparison of expected and actual BAM file outputs of a workflow, the [pysam](https://pysam.readthedocs.io/) library is required.
+Some optional features of pytest-wdl have additional dependencies that are loaded on-demand.
 
-The following plugins require an "extras" installation:
+The plugins that have extra dependencies are:
 
-- Data types
-    - bam
-- URL schemes
-    - dx (DNAnexus)
-- Other
-    - progress (show progress bars when downloading files)
+* dx: Support for DNAnexus file storage, and for the dxWDL executor.
+* bam: More intelligent comparison of expected and actual BAM file outputs of a workflow than just comparing MD5 checksums.
+* progress: Show progress bars when downloading remote files.
 
-To install the dependencies for a data type that has extra dependencies:
+To install a plugin's dependencies:
 
 ```
-$ pip install pytest-wdl[<data_type>]
+$ pip install pytest-wdl[<plugin>]
 ```
 
 To do this locally, you can clone the repo and run:
@@ -68,6 +68,15 @@ To install pytest-wdl and **all** extras dependencies:
 $ pip install pytest-wdl[all]
 ```
 
+## Configuration
+
+Some minimal configuration is required to get started with pytest-wdl. Configuration can be provided via environment variables, fixture functions, and/or a config file. To get started, copy one of the following example config files to `$HOME/.pytest_wdl_config.json` and modify as necessary:
+
+* [simple](examples/simple.pytest_wdl_config.json): Uses only the miniwdl executor
+* [more complex](examples/complex.pytest_wdl_config.json): Uses both miniwdl and Cromwell; shows how to configure proxies and headers for accessing remote data files in a private repository
+
+See the [manual](https://pytest-wdl.readthedocs.io/en/stable/manual.html#configuration) for more details on configuring pytest-wdl.
+
 ## Usage
 
 The pytest-wdl plugin provides a set of fixtures for use with pytest. Here is a quick example:
@@ -94,6 +103,8 @@ This test will execute a workflow (such as the following one) with the specified
 # variant_caller.wdl
 version 1.0
 
+import "variant_caller.wdl"
+
 struct Index {
   File fasta
   String organism
@@ -105,7 +116,14 @@ workflow call_variants {
     File bai
     Index index
   }
-  ...
+  
+  call variant_caller.variant_caller {
+    input:
+      bam=bam,
+      bai=bai,
+      index=index
+  }
+
   output {
     File vcf = variant_caller.vcf
   }
@@ -137,11 +155,13 @@ For details, [read the docs](https://pytest-wdl.readthedocs.io).
 
 ## Contributing
 
-To develop pytest-wdl, clone the repository and install all the dependencies:
+If you would like to contribute code to pytest-wdl, please fork the repository and submit your changes via pull request.
+
+To get started developing pytest-wdl, first install all the development requirements:
 
 ```commandline
 $ git clone https://github.com/EliLillyCo/pytest-wdl.git
-$ pip install -r requirements.txt
+$ make install_development_requirements
 ```
 
 To run the full build and unit tests, run: