diff --git a/requirements-wdl.txt b/requirements-wdl.txt index d2e1ec1391..fcbc25a76c 100644 --- a/requirements-wdl.txt +++ b/requirements-wdl.txt @@ -1,2 +1,2 @@ -miniwdl==1.13.1 +miniwdl==1.15.0 wdlparse==0.1.0 diff --git a/src/toil/test/wdl/wdltoil_test.py b/src/toil/test/wdl/wdltoil_test.py index f6ddf01023..144842066e 100644 --- a/src/toil/test/wdl/wdltoil_test.py +++ b/src/toil/test/wdl/wdltoil_test.py @@ -49,58 +49,28 @@ # These tests (in the same order as in SPEC.md) are known to fail WDL_11_UNIT_TESTS_UNSUPPORTED_BY_TOIL = [ "test_object", # Objects are not supported - "map_to_struct", # miniwdl cannot coerce map to struct, https://github.com/chanzuckerberg/miniwdl/issues/712 - "relative_and_absolute_task", # needs root to run "test_gpu_task", # needs gpu to run, else warning - "hisat2_task", # This needs way too many resources (and actually doesn't work?), see https://github.com/DataBiosphere/wdl-conformance-tests/blob/2d617b703a33791f75f30a9db43c3740a499cd89/README_UNIT.md?plain=1#L8 - "gatk_haplotype_caller_task", # same as above "input_ref_call", # Inputs refering into workflow body not yet implemented: see https://github.com/DataBiosphere/toil/issues/4993 "call_imported", # Same as input_ref_call since it imports it - "call_imported_task", # Same as input_ref_call since it imports it - "test_sub", # MiniWDL does not handle metacharacters properly when running regex, https://github.com/chanzuckerberg/miniwdl/issues/709 - "read_bool_task", # miniwdl bug, see https://github.com/chanzuckerberg/miniwdl/issues/701 "write_json_fail", # miniwdl (and toil) bug, unserializable json is serialized, see https://github.com/chanzuckerberg/miniwdl/issues/702 "read_object_task", # object not supported "read_objects_task", # object not supported "write_object_task", # object not supported "write_objects_task", # object not supported - "test_transpose", # miniwdl bug, see https://github.com/chanzuckerberg/miniwdl/issues/699 - "test_as_map_fail", # miniwdl bug, evalerror, see https://github.com/chanzuckerberg/miniwdl/issues/700 - "test_collect_by_key", # same as test_as_map_ ] WDL_12_UNIT_TESTS_UNSUPPORTED_BY_TOIL = WDL_11_UNIT_TESTS_UNSUPPORTED_BY_TOIL + [ - "relative_paths_context", # Toil can't yet resolve File coercion at task scope relative to task file. + "relative_paths_context", # Workflow succeeds but output content does not match expected ("Expected and result do not match!") "file_directory_equality", # String to Directory coercion not yet implemented. - "single_return_code_task", # MiniWDL 1.13.1 only knows returnCodes and not return_codes. - "all_return_codes_task", # MiniWDL 1.13.1 only knows returnCodes and not return_codes. - "test_runtime_info_task", # MiniWDL 1.13.1 can't yet expose the task global. - "placeholder_none", # 'outputs' section expected 1 results (['placeholder_none.s']), got 0 instead ([]) with exit code 1 - "person_struct_task", # Doesn't work as written in the spec; see https://github.com/openwdl/wdl/issues/739 - "import_structs", # Feature not yet implemented? - "environment_variable_should_echo", # Ln 14 Col 45: Unexpected token STRING1_FRAGMENT - "outputs_task", # 'outputs' section expected 2 results (['outputs.threshold', 'outputs.two_csvs']), got 3 instead (['outputs.two_csvs', 'outputs.csvs', 'outputs.threshold']) with exit code 0 - "glob_task", # 'outputs' section expected 1 results (['glob.last_file_contents']), got 2 instead (['glob.last_file_contents', 'glob.outfiles']) with exit code 0 - "test_hints_task", # Test is written as if the file has 3 lines, but it really has 2. See https://github.com/openwdl/wdl/issues/741 - "input_hint_task", # Missing outputs in test definition: https://github.com/openwdl/wdl/issues/740 - "test_allow_nested_inputs", # Ln 27 Col 3: Unexpected token HINTS - "multi_nested_inputs", # Ln 8 Col 9: Unexpected token STRING1_FRAGMENT - "allow_nested", # Ln 32 Col 9: Unexpected token STRING1_FRAGMENT - "test_find_task", # Ln 9 Col 22: No such function: find - "test_matches_task", # Ln 7 Col 29: No such function: matches - "change_extension_task", # 'outputs' section expected 2 results (['change_extension.data', 'change_extension.index']), got 3 instead (['change_extension.index', 'change_extension.data', 'change_extension.data_file']) with exit code 0 - "join_paths_task", # Ln 14 Col 15: No such function: join_paths - "gen_files_task", # 'outputs' section expected 1 results (['gen_files.glob_len']), got 2 instead (['gen_files.glob_len', 'gen_files.files']) with exit code 0 - "file_sizes_task", # WDL.Error.StaticTypeMismatch: Expected File? instead of Map[String,Pair[Int,File?]] + "single_return_code_task", # Task correctly exits 1 as declared via requirements.return_codes, but Toil/miniwdl still requires exit code 0 - return_codes override is not honored + "all_return_codes_task", # Task correctly exits 42 as declared via requirements.return_codes: "*", but Toil/miniwdl still requires exit code 0 - return_codes override is not honored + "test_runtime_info_task", # WDL.Error.EvalError from a KeyError; task runtime global still not fully exposed + "placeholder_none", # WDL.Error.EvalError: select_first() given empty or all-null array; prevent this or append a default value + "environment_variable_should_echo", # Now parses; Expected and result do not match! + "multi_nested_inputs", # Workflow did not fail! Fails for toil-wdl and miniwdl + "join_paths_task", # PermissionError: [Errno 13] Permission denied: '/usr/bin/sudo' + "file_sizes_task", # WDL.Error.InputError: cannot coerce Map[String,Pair[Int,File?]] to Array[File?] "read_tsv_task", # Ln 21 Col 5: Unknown type Object - "write_tsv_task", # Ln 28 Col 16: write_tsv expects 1 argument(s) - "test_contains", # Ln 25 Col 22: No such function: contains - "chunk_array", # Ln 8 Col 17: No such function: chunk - "test_select_first", # Ln 14 Col 17: select_first expects 1 argument(s) - "test_keys", # Ln 32 Col 36: Expected Map[Any,Any] instead of Name - "test_contains_key", # Ln 18 Col 20: No such function: contains_key - "test_values", # Ln 28 Col 20: No such function: values - "test_length", # length() isn't implemented for maps and strings yet ] diff --git a/src/toil/wdl/wdltoil.py b/src/toil/wdl/wdltoil.py index 883cc524af..6e0d19fc3f 100755 --- a/src/toil/wdl/wdltoil.py +++ b/src/toil/wdl/wdltoil.py @@ -858,6 +858,7 @@ def fill_execution_cache( output_bindings: WDLBindings, file_store: AbstractFileStore, wdl_options: WDLContext, + input_bindings: WDLBindings, miniwdl_logger: logging.Logger | None = None, miniwdl_config: WDL.runtime.config.Loader | None = None, ) -> WDLBindings: @@ -946,7 +947,12 @@ def assign_shared_fs_path(inode: AnyINode) -> AnyINode: ) # Save the bindings to the cache, representing all files with their shared filesystem paths. - miniwdl_cache.put(cache_key, view_shared_fs_paths(output_bindings)) + miniwdl_cache.put( + cache_key, + view_shared_fs_paths(output_bindings), + inputs=view_shared_fs_paths(input_bindings), + add_paths=WDL.runtime.cache.CallCacheAddPaths(), + ) logger.debug("Saved result to cache under %s", cache_key) # Keep using the transformed bindings so that later tasks use @@ -2415,6 +2421,8 @@ def wrapper(v: WDL.Value.Base) -> WDL.Value.File: WDL.Env.Bindings( WDL.Env.Binding("file", WDL.Value.File(exported_path)) ), + inputs=file_input_bindings, + add_paths=WDL.runtime.cache.CallCacheAddPaths(), ) # Apply the shared filesystem path to the virtualized file @@ -4531,6 +4539,7 @@ def get_path_in_container(inode: AnyINode) -> AnyINode | None: output_bindings, file_store, self._wdl_options, + unwrap(self._task_internal_bindings), miniwdl_logger=miniwdl_logger, miniwdl_config=miniwdl_config, ) @@ -5632,6 +5641,7 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]: self._enclosing_bindings, wdl_options=self._wdl_options, cache_key=cache_key, + input_bindings=bindings, local=True, ) sink.addFollowOn(outputs_job) @@ -5654,6 +5664,7 @@ def __init__( enclosing_bindings: WDLBindings, wdl_options: WDLContext, cache_key: str | None = None, + input_bindings: Promised[WDLBindings] | None = None, **kwargs: Any, ): """ @@ -5667,6 +5678,9 @@ def __init__( :param cache_key: If set and storing into the call cache is on, will cache the workflow execution result under the given key in a MiniWDL-compatible way. + + :param input_bindings: The workflow call's input bindings, matching + what cache_key was derived from. Required if cache_key is set. """ super().__init__(wdl_options=wdl_options, **kwargs) @@ -5674,6 +5688,7 @@ def __init__( self._enclosing_bindings = enclosing_bindings self._workflow = workflow self._cache_key = cache_key + self._input_bindings = input_bindings @report_wdl_errors("evaluate outputs") def run(self, file_store: AbstractFileStore) -> WDLBindings: @@ -5757,8 +5772,13 @@ def run(self, file_store: AbstractFileStore) -> WDLBindings: output_bindings = virtualize_inodes(output_bindings, standard_library) if self._cache_key is not None: + assert self._input_bindings is not None output_bindings = fill_execution_cache( - self._cache_key, output_bindings, file_store, self._wdl_options + self._cache_key, + output_bindings, + file_store, + self._wdl_options, + unwrap(self._input_bindings), ) # Let Files that are not output or available outside the call go out of