Merge branch 'dxcompiler' of https://github.com/EliLillyCo/pytest-wdl …

…into dxcompiler

.travis.yml

@@ -24,6 +24,7 @@ before_script:
 - export CROMWELL_JAR=/tmp/cromwell-53.1.jar
 script:
 - make test
+- make quickstart
 - make test_release_setup
 after_success:
 - bash <(curl -s https://codecov.io/bash)

CHANGES.md

@@ -2,6 +2,8 @@
 
 ## Development
 
+* Updated `miniwdl` dependency to 0.9.0
+* Fix #144 - Pair type not supported by miniwdl executor
 
 ## v1.4.1 (2020.11.17)
 * Replaces remote file localization method for adding HTTP headers to only add headers on initial request and not redirects.

Makefile

@@ -75,3 +75,6 @@ release: clean tag
 		-H "Authorization: token $(token)" \
 		https://api.github.com/repos/$(repo)/releases \
 		-d '{"tag_name":"$(version)","target_commitish": "main","name": "$(version)","body": "$(desc)","draft": false,"prerelease": false}'
+
+quickstart:
+	cd examples/quickstart && pytest -s -vv --show-capture=all

README.md

@@ -11,7 +11,7 @@ This package is a plugin for the [pytest](https://docs.pytest.org/en/latest/) un
 * Python 3.6+
 * At least one of the supported workflow engines:
     * [Miniwdl](https://github.com/chanzuckerberg/miniwdl) - automatically installed as a dependency of pytest-wdl
-    * [Cromwell](https://github.com/broadinstitute/cromwell/releases/tag/53.1) JAR file
+    * [Cromwell](https://github.com/broadinstitute/cromwell/releases) JAR file
         * **Cromwell Server**: Any Cromwell instance remotely running in Server mode
     * [dxWDL](https://github.com/dnanexus/dxWDL) JAR file
 * Java-based workflow engines (e.g. Cromwell and dxWDL) require a Java runtime (typically 1.8+)
@@ -196,6 +196,8 @@ To run the full build and unit tests, run:
 $ make
 ```
 
+Note that some tests will be skipped if you are not logged into a [DNAnexus](https://platform.dnanexus.com) account.
+
 ## Support
 
 pytest-wdl is *not* an official product of Eli Lilly or DNAnexus. Please do *not* contact these companies (or any employees thereof) for support. To report a bug or feature request, please open an issue in the [issue tracker](https://github.com/EliLillyCo/pytest-wdl/issues).

docs/requirements.txt

@@ -1,5 +1,5 @@
-Sphinx>=2.1.2
-recommonmark>=0.5.0
-sphinx_rtd_theme>=0.4.3
-sphinx-markdown-tables>=0.0.9
-sphinxcontrib-apidoc>=0.3.0
+Sphinx==2.1.2
+recommonmark==0.5.0
+sphinx_rtd_theme==0.4.3
+sphinx-markdown-tables==0.0.9
+sphinxcontrib-apidoc==0.3.0

examples/config/complex.pytest_wdl_config.json

@@ -22,7 +22,7 @@
     "cromwell": {
       "java_bin": "/usr/local/bin/java",
       "java_args": "-Xmx8g -Ddocker.hash-lookup.enabled=false",
-      "cromwell_jar_file": "/usr/local/opt/cromwell/Cromwell-48.jar"
+      "cromwell_jar_file": "/usr/local/opt/cromwell/Cromwell-54.jar"
     }
   }
 }

pytest_wdl/config.py

@@ -22,7 +22,7 @@
 from pytest_wdl.utils import ensure_path, env_map
 
 try:
-    import yaml
+    from ruamel import yaml
 except ImportError:
     yaml = None
 
@@ -89,7 +89,9 @@ def __init__(
         if config_file:
             with open(config_file, "rt") as inp:
                 if yaml and config_file.suffix == ".yaml":
-                    defaults = yaml.load(inp)
+                    yaml_loader = yaml.YAML(typ="safe")
+                    yaml_loader.default_flow_style = False
+                    defaults = yaml_loader.load(inp)
                 else:
                     defaults = json.load(inp)
         else:
@@ -206,7 +208,9 @@ def save(self, path: Path) -> None:
         d = self.as_dict()
         with open(path, "wt") as out:
             if yaml and path.suffix == ".yaml":
-                yaml.dump(d, out)
+                yaml_dumper = yaml.YAML(typ="safe")
+                yaml_dumper.default_flow_style = False
+                yaml_dumper.dump(d, out)
             else:
                 json.dump(d, out)
 

pytest_wdl/data_types/vcf.py

@@ -54,7 +54,9 @@ def diff_vcf_columns(file1: Path, file2: Path, compare_phase: bool = False) -> i
     with tempdir() as temp:
         def make_comparable(infile, outfile):
             cmd = ["grep -vE '^#'", "cut -f 1-5,7,10", "cut -d ':' -f 1"]
-            output = subby.sub(cmd, stdin=infile)
+            # if the VCF file is empty or (for some reason) has no headers,
+            # grep will exit with return code 1
+            output = subby.sub(cmd, stdin=infile, allowed_return_codes=(0, 1))
             with open(outfile, "wt") as out:
                 if compare_phase:
                     out.write(output)

pytest_wdl/executors/__init__.py

@@ -24,6 +24,7 @@
     ensure_path, safe_string, find_executable_path, find_in_classpath
 )
 
+import WDL
 from WDL import Document, Error, Tree
 
 
@@ -317,7 +318,7 @@ def parse_wdl(
     check_quant: bool = False,
     **_
 ) -> Document:
-    return Tree.load(
+    return WDL.load(
         str(wdl_path),
         path=[str(path) for path in import_dirs],
         check_quant=check_quant

pytest_wdl/executors/cromwell_server.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+import time
 from typing import IO, Optional, Sequence, Union
 
 import requests
@@ -206,12 +207,35 @@ def _poll_until_terminal(
         run_id: str,
         target: str,
         inputs_dict: Optional[dict] = None,
-        timeout: int = DEFAULT_POLLING_TIMEOUT
+        timeout: int = DEFAULT_POLLING_TIMEOUT,
+        num_retries: int = 5,
+        retry_interval: int = 1
     ):
         def get_status(status_url):
-            with requests.get(status_url, auth=self._auth) as rsp:
-                status_dict = self._resp_to_json(rsp, target, inputs_dict)
-                return status_dict.get("status") in TERMINAL_STATES
+            # Cromwell may return a 404 error, especially right after
+            # calling the run API endpoint - retry here several times
+            # and wait after each 404 error
+            # see https://github.com/EliLillyCo/pytest-wdl/issues/155#issuecomment-750438858
+            for i in range(num_retries):
+                with requests.get(status_url, auth=self._auth) as rsp:
+                    if rsp.status_code == 404:
+                        time.sleep(retry_interval)
+                    else:
+                        status_dict = self._resp_to_json(rsp, target, inputs_dict)
+                        return status_dict.get("status") in TERMINAL_STATES
+            else:
+                status_err = (
+                    f"Failed to get response from {status_url} within "
+                    f"{num_retries * retry_interval} seconds"
+                )
+                LOG.error(status_err)
+                raise ExecutionFailedError(
+                    executor="cromwell-server",
+                    target=target,
+                    status="Failed",
+                    inputs=inputs_dict,
+                    msg=status_err
+                )
 
         try:
             poll(
@@ -221,14 +245,12 @@ def get_status(status_url):
                 timeout=timeout
             )
         except PollingException:
-            LOG.exception(f"Encountered timeout for run with id {run_id}")
-
-            error_kwargs = {
-                "executor": "cromwell-server",
-                "target": target,
-                "status": "Failed",
-                "inputs": inputs_dict,
-                "msg": f"Encountered timeout for run with id {run_id}",
-            }
-
-            raise ExecutionFailedError(**error_kwargs)
+            polling_err = f"Encountered timeout for run with id {run_id}"
+            LOG.exception(polling_err)
+            raise ExecutionFailedError(
+                executor="cromwell-server",
+                target=target,
+                status="Failed",
+                inputs=inputs_dict,
+                msg=polling_err
+            )