diff --git a/Makefile b/Makefile index 28d9b8219..9b864ed19 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py -DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \ +DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox>4' auto-walrus \ isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\ -rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \ diff --git a/cwltool/builder.py b/cwltool/builder.py index cf6d6d5ac..5c615a3ec 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -9,6 +9,7 @@ from cwl_utils import expression from cwl_utils.file_formats import check_format +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType from mypy_extensions import mypyc_attr from rdflib import Graph from ruamel.yaml.comments import CommentedMap @@ -26,8 +27,6 @@ from .stdfsaccess import StdFsAccess from .utils import ( CONTENT_LIMIT, - CWLObjectType, - CWLOutputType, HasReqsHints, LoadListingType, aslist, @@ -95,7 +94,7 @@ class Builder(HasReqsHints): def __init__( self, job: CWLObjectType, - files: list[CWLObjectType], + files: MutableSequence[CWLFileType | CWLDirectoryType], bindings: list[CWLObjectType], schemaDefs: MutableMapping[str, CWLObjectType], names: Names, @@ -166,7 +165,7 @@ def build_job_script(self, commands: list[str]) -> str | None: return self.job_script_provider.build_job_script(self, commands) return None - def _capture_files(self, f: CWLObjectType) -> CWLObjectType: + def _capture_files(self, f: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType: self.files.append(f) return f @@ -356,13 +355,13 @@ def bind_input( ) binding = {} - def _capture_files(f: CWLObjectType) -> CWLObjectType: + def _capture_files(f: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType: self.files.append(f) return f if schema["type"] == "org.w3id.cwl.cwl.File": - datum = cast(CWLObjectType, datum) - self.files.append(datum) + file_datum = cast(CWLFileType, datum) + self.files.append(file_datum) loadContents_sourceline: ( None | MutableMapping[str, str | list[int]] | CWLObjectType @@ -380,14 +379,16 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: debug, ): try: - with self.fs_access.open(cast(str, datum["location"]), "rb") as f2: - datum["contents"] = content_limit_respected_read(f2) + with self.fs_access.open(file_datum["location"], "rb") as f2: + file_datum["contents"] = content_limit_respected_read(f2) except Exception as e: - raise Exception("Reading {}\n{}".format(datum["location"], e)) from e + raise Exception( + "Reading {}\n{}".format(file_datum["location"], e) + ) from e if "secondaryFiles" in schema: - if "secondaryFiles" not in datum: - datum["secondaryFiles"] = [] + if "secondaryFiles" not in file_datum: + file_datum["secondaryFiles"] = [] sf_schema = aslist(schema["secondaryFiles"]) elif not discover_secondaryFiles: sf_schema = [] # trust the inputs @@ -396,7 +397,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: for num, sf_entry in enumerate(sf_schema): if "required" in sf_entry and sf_entry["required"] is not None: - required_result = self.do_eval(sf_entry["required"], context=datum) + required_result = self.do_eval(sf_entry["required"], context=file_datum) if not (isinstance(required_result, bool) or required_result is None): if sf_schema == schema["secondaryFiles"]: sf_item: Any = sf_schema[num] @@ -418,7 +419,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: if "$(" in sf_entry["pattern"] or "${" in sf_entry["pattern"]: sfpath = self.do_eval(sf_entry["pattern"], context=datum) else: - sfpath = substitute(cast(str, datum["basename"]), sf_entry["pattern"]) + sfpath = substitute(file_datum["basename"], sf_entry["pattern"]) for sfname in aslist(sfpath): if not sfname: @@ -426,7 +427,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: found = False if isinstance(sfname, str): - d_location = cast(str, datum["location"]) + d_location = file_datum["location"] if "/" in d_location: sf_location = ( d_location[0 : d_location.rindex("/") + 1] + sfname @@ -435,6 +436,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: sf_location = d_location + sfname sfbasename = sfname elif isinstance(sfname, MutableMapping): + sfname = cast(CWLFileType | CWLDirectoryType, sfname) sf_location = sfname["location"] sfbasename = sfname["basename"] else: @@ -447,10 +449,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: f"{type(sfname)!r} from {sf_entry['pattern']!r}." ) - for d in cast( - MutableSequence[MutableMapping[str, str]], - datum["secondaryFiles"], - ): + for d in file_datum["secondaryFiles"]: if not d.get("basename"): d["basename"] = d["location"][d["location"].rindex("/") + 1 :] if d["basename"] == sfbasename: @@ -459,8 +458,8 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: if not found: def addsf( - files: MutableSequence[CWLObjectType], - newsf: CWLObjectType, + files: MutableSequence[CWLFileType | CWLDirectoryType], + newsf: CWLFileType | CWLDirectoryType, ) -> None: for f in files: if f["location"] == newsf["location"]: @@ -470,23 +469,19 @@ def addsf( if isinstance(sfname, MutableMapping): addsf( - cast( - MutableSequence[CWLObjectType], - datum["secondaryFiles"], - ), - sfname, + file_datum["secondaryFiles"], + cast(CWLFileType | CWLDirectoryType, sfname), ) elif discover_secondaryFiles and self.fs_access.exists(sf_location): addsf( - cast( - MutableSequence[CWLObjectType], - datum["secondaryFiles"], + file_datum["secondaryFiles"], + CWLFileType( + **{ + "location": sf_location, + "basename": sfname, + "class": "File", + } ), - { - "location": sf_location, - "basename": sfname, - "class": "File", - }, ) elif sf_required: raise SourceLine( @@ -496,12 +491,10 @@ def addsf( debug, ).makeError( "Missing required secondary file '%s' from file object: %s" - % (sfname, json_dumps(datum, indent=4)) + % (sfname, json_dumps(file_datum, indent=4)) ) - normalizeFilesDirs( - cast(MutableSequence[CWLObjectType], datum["secondaryFiles"]) - ) + normalizeFilesDirs(file_datum["secondaryFiles"]) if "format" in schema: eval_format: Any = self.do_eval(schema["format"]) @@ -546,7 +539,7 @@ def addsf( ) try: check_format( - datum, + file_datum, evaluated_format, self.formatgraph, ) @@ -557,21 +550,21 @@ def addsf( ) from ve visit_class( - datum.get("secondaryFiles", []), + file_datum.get("secondaryFiles", []), ("File", "Directory"), self._capture_files, ) if schema["type"] == "org.w3id.cwl.cwl.Directory": - datum = cast(CWLObjectType, datum) + dir_datum = cast(CWLDirectoryType, datum) ll = schema.get("loadListing") or self.loadListing if ll and ll != "no_listing": get_listing( self.fs_access, - datum, + dir_datum, (ll == "deep_listing"), ) - self.files.append(datum) + self.files.append(dir_datum) if schema["type"] == "Any": visit_class(datum, ("File", "Directory"), self._capture_files) @@ -596,9 +589,7 @@ def tostr(self, value: MutableMapping[str, str] | Any) -> str: match value: case {"class": "File" | "Directory" as class_name, **rest}: if "path" not in rest: - raise WorkflowException( - '{} object missing "path": {}'.format(class_name, value) - ) + raise WorkflowException(f'{class_name} object missing "path": {value}') return str(rest["path"]) case ScalarFloat(): rep = RoundTripRepresenter() diff --git a/cwltool/checker.py b/cwltool/checker.py index d81642e1d..c30a0d0b5 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -3,6 +3,7 @@ from collections.abc import Iterator, MutableMapping, MutableSequence, Sized from typing import Any, Literal, NamedTuple, Optional, Union, cast +from cwl_utils.types import CWLObjectType, CWLOutputType, SinkType from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno from schema_salad.utils import json_dumps @@ -10,7 +11,7 @@ from .errors import WorkflowException from .loghandler import _logger from .process import shortname -from .utils import CWLObjectType, CWLOutputType, SinkType, aslist +from .utils import aslist def _get_type(tp: Any) -> Any: @@ -21,8 +22,8 @@ def _get_type(tp: Any) -> Any: def check_types( - srctype: SinkType, - sinktype: SinkType, + srctype: SinkType | None, + sinktype: SinkType | None, linkMerge: str | None, valueFrom: str | None, ) -> Literal["pass"] | Literal["warning"] | Literal["exception"]: @@ -55,7 +56,7 @@ def check_types( raise WorkflowException(f"Unrecognized linkMerge enum {linkMerge!r}") -def merge_flatten_type(src: SinkType) -> CWLOutputType: +def merge_flatten_type(src: SinkType | None) -> CWLOutputType | None: """Return the merge flattened type of the source type.""" match src: case MutableSequence(): @@ -66,7 +67,9 @@ def merge_flatten_type(src: SinkType) -> CWLOutputType: return {"items": src, "type": "array"} -def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool = False) -> bool: +def can_assign_src_to_sink( + src: SinkType | None, sink: SinkType | None, strict: bool = False +) -> bool: """ Check for identical type specifications, ignoring extra keys like inputBinding. @@ -84,8 +87,8 @@ def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool = return False if src["type"] == "array" and sink["type"] == "array": return can_assign_src_to_sink( - cast(MutableSequence[CWLOutputType], src["items"]), - cast(MutableSequence[CWLOutputType], sink["items"]), + cast(MutableSequence[CWLOutputType | None], src["items"]), + cast(MutableSequence[CWLOutputType | None], sink["items"]), strict, ) if src["type"] == "record" and sink["type"] == "record": diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index a9d5e57e2..499601979 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -14,7 +14,6 @@ import urllib.parse from collections.abc import ( Generator, - Iterable, Mapping, MutableMapping, MutableSequence, @@ -22,8 +21,17 @@ from enum import Enum from functools import cmp_to_key, partial from re import Pattern -from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast +from typing import Any, Optional, TYPE_CHECKING, TextIO, Union, cast +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + is_directory, + is_file, + is_file_or_directory, +) from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import RecordSchema @@ -33,12 +41,7 @@ from schema_salad.utils import json_dumps from schema_salad.validate import validate_ex -from .builder import ( - INPUT_OBJ_VOCAB, - Builder, - content_limit_respected_read_bytes, - substitute, -) +from .builder import Builder, INPUT_OBJ_VOCAB, content_limit_respected_read_bytes, substitute from .context import LoadingContext, RuntimeContext, getdefault from .docker import DockerCommandLineJob, PodmanCommandLineJob from .errors import UnsupportedRequirement, WorkflowException @@ -60,9 +63,6 @@ from .udocker import UDockerCommandLineJob from .update import ORDERED_VERSIONS, ORIGINAL_CWLVERSION from .utils import ( - CWLObjectType, - CWLOutputType, - DirectoryType, JobsGeneratorType, OutputCallbackType, adjustDirObjs, @@ -178,18 +178,7 @@ def run( try: normalizeFilesDirs(self.builder.job) ev = self.builder.do_eval(self.script) - normalizeFilesDirs( - cast( - Optional[ - Union[ - MutableSequence[MutableMapping[str, Any]], - MutableMapping[str, Any], - DirectoryType, - ] - ], - ev, - ) - ) + normalizeFilesDirs(ev) if self.output_callback: self.output_callback(cast(Optional[CWLObjectType], ev), "success") except WorkflowException as err: @@ -239,7 +228,9 @@ def remove_path(f: CWLObjectType) -> None: del f["path"] -def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> CWLObjectType | None: +def revmap_file( + builder: Builder, outdir: str, f: CWLFileType | CWLDirectoryType +) -> CWLFileType | CWLDirectoryType: """ Remap a file from internal path to external path. @@ -258,18 +249,18 @@ def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> CWLObjectTyp # quoted any further. if "location" in f and "path" not in f: - location = cast(str, f["location"]) + location = f["location"] if location.startswith("file://"): f["path"] = uri_file_path(location) else: - f["location"] = builder.fs_access.join(outdir, cast(str, f["location"])) + f["location"] = builder.fs_access.join(outdir, f["location"]) return f - if "dirname" in f: + if is_file(f) and "dirname" in f: del f["dirname"] if "path" in f: - path = builder.fs_access.join(builder.outdir, cast(str, f["path"])) + path = builder.fs_access.join(builder.outdir, f["path"]) uripath = file_uri(path) del f["path"] @@ -379,9 +370,9 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None: location = cast(str, ob["location"]) if location.startswith("_:"): pass - if ob["class"] == "File" and not fs_access.isfile(location): + if is_file(ob) and not fs_access.isfile(location): raise ValidationException("Does not exist or is not a File: '%s'" % location) - if ob["class"] == "Directory" and not fs_access.isdir(location): + if is_directory(ob) and not fs_access.isdir(location): raise ValidationException("Does not exist or is not a Directory: '%s'" % location) @@ -449,7 +440,7 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: @staticmethod def make_path_mapper( - reffiles: list[CWLObjectType], + reffiles: MutableSequence[CWLFileType | CWLDirectoryType], stagedir: str, runtimeContext: RuntimeContext, separateDirs: bool, @@ -596,10 +587,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: filelist = True for e in entry: - if not isinstance(e, MutableMapping) or e.get("class") not in ( - "File", - "Directory", - ): + if not is_file_or_directory(e): filelist = False break @@ -619,11 +607,8 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: et: CWLObjectType = {} writable = t.get("writable", False) et["writable"] = writable - if isinstance(entry, Mapping) and entry.get("class") in ( - "File", - "Directory", - ): - if writable and "secondaryFiles" in entry: + if is_file_or_directory(entry): + if writable and is_file(entry) and "secondaryFiles" in entry: secFiles = cast(MutableSequence[CWLObjectType], entry["secondaryFiles"]) for sf in secFiles: sf["writable"] = writable @@ -704,7 +689,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: ) ) - if t2["entry"].get("class") not in ("File", "Directory"): + if not is_file_or_directory(t2["entry"]): raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( "Entry at index %s of listing is not a File or Directory object, was %s" % (i, t2) @@ -718,10 +703,10 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: t2entry["writable"] = t2copy.get("writable") t2["entry"] = t2entry - ls[i] = t2["entry"] + ls[i] = cast(CWLObjectType, t2["entry"]) for i, t3 in enumerate(ls): - if t3.get("class") not in ("File", "Directory"): + if not is_file_or_directory(t3): # Check that every item is a File or Directory object now raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( f"Entry at index {i} of listing is not a Dirent, File or " @@ -729,7 +714,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: ) if "basename" not in t3: continue - basename = os.path.normpath(cast(str, t3["basename"])) + basename = os.path.normpath(t3["basename"]) t3["basename"] = basename if basename.startswith("../"): raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( @@ -761,18 +746,19 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: return # Only testing with SourceLine(initialWorkdir, "listing", WorkflowException, debug): - j.generatefiles["listing"] = ls + j.generatefiles["listing"] = cast(MutableSequence[CWLFileType | CWLDirectoryType], ls) for entry in ls: if "basename" in entry: basename = cast(str, entry["basename"]) + entry["basename"] = os.path.basename(basename) dirname = os.path.join(builder.outdir, os.path.dirname(basename)) entry["dirname"] = dirname - entry["basename"] = os.path.basename(basename) - if "secondaryFiles" in entry: - for sec_file in cast( - MutableSequence[CWLObjectType], entry["secondaryFiles"] - ): - sec_file["dirname"] = dirname + if is_file(entry): + if "secondaryFiles" in entry: + for sec_file in cast( + MutableSequence[CWLObjectType], entry["secondaryFiles"] + ): + sec_file["dirname"] = dirname normalizeFilesDirs(entry) self.updatePathmap( cast(Optional[str], entry.get("dirname")) or builder.outdir, @@ -780,7 +766,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: entry, ) - if "listing" in entry: + if is_directory(entry) and "listing" in entry: def remove_dirname(d: CWLObjectType) -> None: if "dirname" in d: @@ -857,10 +843,11 @@ def calc_checksum(location: str) -> str | None: if ( "location" in e and e["location"] == location + and is_file(e) and "checksum" in e and e["checksum"] != "sha1$hash" ): - return cast(str, e["checksum"]) + return e["checksum"] return None def remove_prefix(s: str, prefix: str) -> str: @@ -1019,7 +1006,7 @@ def update_status_output_callback( ) j.stdin = stdin_eval if j.stdin: - reffiles.append({"class": "File", "path": j.stdin}) + reffiles.append(CWLFileType(**{"class": "File", "path": j.stdin})) if self.tool.get("stderr"): with SourceLine(self.tool, "stderr", ValidationException, debug): @@ -1299,7 +1286,7 @@ def collect_output( fs_access: StdFsAccess, compute_checksum: bool = True, ) -> CWLOutputType | None: - r: list[CWLOutputType] = [] + r: MutableSequence[CWLFileType | CWLDirectoryType] = [] empty_and_optional = False debug = _logger.isEnabledFor(logging.DEBUG) result: CWLOutputType | None = None @@ -1347,9 +1334,9 @@ def collect_output( key=cmp_to_key(locale.strcoll), ) r.extend( - cast( - Iterable[CWLOutputType], - [ + [ + cast( + CWLFileType | CWLDirectoryType, { "location": g, "path": fs_access.join( @@ -1360,16 +1347,16 @@ def collect_output( "nameroot": os.path.splitext(decoded_basename)[0], "nameext": os.path.splitext(decoded_basename)[1], "class": "File" if fs_access.isfile(g) else "Directory", - } - for g, decoded_basename in zip( + }, + ) + for g, decoded_basename in zip( + sorted_glob_result, + map( + lambda x: os.path.basename(urllib.parse.unquote(x)), sorted_glob_result, - map( - lambda x: os.path.basename(urllib.parse.unquote(x)), - sorted_glob_result, - ), - ) - ], - ) + ), + ) + ] ) except OSError as e: _logger.warning(str(e), exc_info=builder.debug) @@ -1377,28 +1364,28 @@ def collect_output( _logger.error("Unexpected error from fs_access", exc_info=True) raise - for files in cast(list[dict[str, Optional[CWLOutputType]]], r): + for files in r: rfile = files.copy() revmap(rfile) - if files["class"] == "Directory": + if is_directory(files): ll = binding.get("loadListing") or builder.loadListing if ll and ll != "no_listing": get_listing(fs_access, files, (ll == "deep_listing")) else: if binding.get("loadContents"): - with fs_access.open(cast(str, rfile["location"]), "rb") as f: + with fs_access.open(rfile["location"], "rb") as f: files["contents"] = str( content_limit_respected_read_bytes(f), "utf-8" ) if compute_checksum: - with fs_access.open(cast(str, rfile["location"]), "rb") as f: + with fs_access.open(rfile["location"], "rb") as f: checksum = hashlib.sha1() # nosec contents = f.read(1024 * 1024) while contents != b"": checksum.update(contents) contents = f.read(1024 * 1024) files["checksum"] = "sha1$%s" % checksum.hexdigest() - files["size"] = fs_access.size(cast(str, rfile["location"])) + files["size"] = fs_access.size(rfile["location"]) optional = False single = False diff --git a/cwltool/context.py b/cwltool/context.py index 4e106ac48..44129b40f 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -8,6 +8,7 @@ from collections.abc import Callable, Iterable from typing import IO, TYPE_CHECKING, Any, Literal, Optional, TextIO, Union +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names from schema_salad.ref_resolver import Loader @@ -16,7 +17,7 @@ from .mpi import MpiConfig from .pathmapper import PathMapper from .stdfsaccess import StdFsAccess -from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, HasReqsHints, ResolverType +from .utils import DEFAULT_TMP_PREFIX, HasReqsHints, ResolverType if TYPE_CHECKING: from _typeshed import SupportsWrite diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 86dcb3cdd..ec3635373 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -3,8 +3,9 @@ import subprocess # nosec import xml.dom.minidom # nosec +from cwl_utils.types import CWLObjectType + from .loghandler import _logger -from .utils import CWLObjectType def cuda_version_and_device_count() -> tuple[str, int]: diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index c0aa5ec5b..6658bd54c 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -6,8 +6,15 @@ from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath -from typing import TYPE_CHECKING, Any, cast - +from typing import TYPE_CHECKING, Any, TypedDict, cast + +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + is_directory, + is_file, +) from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity from schema_salad.sourceline import SourceLine @@ -17,7 +24,7 @@ from ..loghandler import _logger from ..process import Process, shortname from ..stdfsaccess import StdFsAccess -from ..utils import CWLObjectType, JobsType, get_listing, posix_path, versionstring +from ..utils import JobsType, get_listing, posix_path, versionstring from ..workflow_job import WorkflowJob from .provenance_constants import ( ACCOUNT_UUID, @@ -40,6 +47,16 @@ if TYPE_CHECKING: from .ro import ResearchObject +CWLArtifact = TypedDict("CWLArtifact", {"@id": str}) + + +class _CWLDirectoryArtifact(CWLArtifact, CWLDirectoryType): + pass + + +class _CWLFileArtifact(CWLArtifact, CWLFileType): + pass + def copy_job_order(job: Process | JobsType, job_order_object: CWLObjectType) -> CWLObjectType: """Create copy of job object for provenance.""" @@ -243,15 +260,15 @@ def record_process_end( self.generate_output_prov(outputs, process_run_id, process_name) self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) - def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: + def declare_file(self, value: _CWLFileArtifact) -> tuple[ProvEntity, ProvEntity, str]: """Construct a FileEntity for the given CWL File object.""" - if value["class"] != "File": + if not is_file(value): raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename entity: ProvEntity | None = None checksum = None if "checksum" in value: - csum = cast(str, value["checksum"]) + csum = value["checksum"] (method, checksum) = csum.split("$", 1) if method == SHA1 and self.research_object.has_data_file(checksum): entity = self.document.entity("data:" + checksum) @@ -269,7 +286,7 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st if not entity and "contents" in value: # Anonymous file, add content as string - entity, checksum = self.declare_string(cast(str, value["contents"])) + entity, checksum = self.declare_string(value["contents"]) # By here one of them should have worked! if not entity or not checksum: @@ -279,7 +296,7 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st # secondaryFiles. Note that multiple uses of a file might thus record # different names for the same entity, so we'll # make/track a specialized entity by UUID - file_id = cast(str, value.setdefault("@id", uuid.uuid4().urn)) + file_id = value.setdefault("@id", uuid.uuid4().urn) # A specialized entity that has just these names file_entity = self.document.entity( file_id, @@ -287,20 +304,20 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st ) if "basename" in value: - file_entity.add_attributes({CWLPROV["basename"]: cast(str, value["basename"])}) + file_entity.add_attributes({CWLPROV["basename"]: value["basename"]}) if "nameroot" in value: - file_entity.add_attributes({CWLPROV["nameroot"]: cast(str, value["nameroot"])}) + file_entity.add_attributes({CWLPROV["nameroot"]: value["nameroot"]}) if "nameext" in value: - file_entity.add_attributes({CWLPROV["nameext"]: cast(str, value["nameext"])}) + file_entity.add_attributes({CWLPROV["nameext"]: value["nameext"]}) self.document.specializationOf(file_entity, entity) # Check for secondaries - for sec in cast(MutableSequence[CWLObjectType], value.get("secondaryFiles", [])): + for sec in value.get("secondaryFiles", []): # TODO: Record these in a specializationOf entity with UUID? - if sec["class"] == "File": - (sec_entity, _, _) = self.declare_file(sec) - elif sec["class"] == "Directory": - sec_entity = self.declare_directory(sec) + if is_file(sec): + (sec_entity, _, _) = self.declare_file(cast(_CWLFileArtifact, sec)) + elif is_directory(sec): + sec_entity = self.declare_directory(cast(_CWLDirectoryArtifact, sec)) else: raise ValueError(f"Got unexpected secondaryFiles value: {sec}") # We don't know how/when/where the secondary file was generated, @@ -315,14 +332,14 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st return file_entity, entity, checksum - def declare_directory(self, value: CWLObjectType) -> ProvEntity: + def declare_directory(self, value: _CWLDirectoryArtifact) -> ProvEntity: """Register any nested files/directories.""" # FIXME: Calculate a hash-like identifier for directory # so we get same value if it's the same filenames/hashes # in a different location. # For now, mint a new UUID to identify this directory, but # attempt to keep it inside the value dictionary - dir_id = cast(str, value.setdefault("@id", uuid.uuid4().urn)) + dir_id = value.setdefault("@id", uuid.uuid4().urn) # New annotation file to keep the ORE Folder listing ore_doc_fn = dir_id.replace("urn:uuid:", "directory-") + ".ttl" @@ -339,7 +356,7 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: ) if "basename" in value: - coll.add_attributes({CWLPROV["basename"]: cast(str, value["basename"])}) + coll.add_attributes({CWLPROV["basename"]: value["basename"]}) # ORE description of ro:Folder, saved separately coll_b = dir_bundle.entity( @@ -475,11 +492,11 @@ def declare_artefact(self, value: Any) -> ProvEntity: # Base case - we found a File we need to update case {"class": "File"}: - entity = self.declare_file(value)[0] + entity = self.declare_file(cast(_CWLFileArtifact, value))[0] value["@id"] = entity.identifier.uri return entity case {"class": "Directory"}: - entity = self.declare_directory(value) + entity = self.declare_directory(cast(_CWLDirectoryArtifact, value)) value["@id"] = entity.identifier.uri return entity case {**rest}: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 28b7c86df..392b183f5 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -13,18 +13,20 @@ from typing import IO, TYPE_CHECKING, Any, Optional, cast import prov.model as provM +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + is_directory, + is_file, + is_file_or_directory, +) from prov.model import ProvDocument from ..loghandler import _logger from ..stdfsaccess import StdFsAccess -from ..utils import ( - CWLObjectType, - CWLOutputType, - create_tmp_dir, - local_path, - posix_path, - versionstring, -) +from ..utils import create_tmp_dir, local_path, posix_path, versionstring from . import Aggregate, Annotation, AuthoredBy, _valid_orcid, _whoami, checksum_copy from .provenance_constants import ( ACCOUNT_UUID, @@ -495,7 +497,7 @@ def _authored_by(self) -> AuthoredBy | None: return authored_by return None - def generate_snapshot(self, prov_dep: CWLObjectType) -> None: + def generate_snapshot(self, prov_dep: CWLFileType | CWLDirectoryType) -> None: """Copy all of the CWL files to the snapshot/ directory.""" self.self_check() for key, value in prov_dep.items(): @@ -521,8 +523,8 @@ def generate_snapshot(self, prov_dep: CWLObjectType) -> None: except PermissionError: pass # FIXME: avoids duplicate snapshotting; need better solution elif key in ("secondaryFiles", "listing"): - for files in cast(MutableSequence[CWLObjectType], value): - if isinstance(files, MutableMapping): + for files in cast(MutableSequence[CWLFileType | CWLDirectoryType], value): + if is_file_or_directory(files): self.generate_snapshot(files) else: pass @@ -635,17 +637,17 @@ def _add_to_bagit(self, rel_path: str, **checksums: str) -> None: def _relativise_files( self, - structure: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType], + structure: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType] | None, ) -> None: """Save any file objects into the RO and update the local paths.""" # Base case - we found a File we need to update _logger.debug("[provenance] Relativising: %s", structure) if isinstance(structure, MutableMapping): - if structure.get("class") == "File": + if is_file(structure): relative_path: str | PurePosixPath | None = None if "checksum" in structure: - raw_checksum = cast(str, structure["checksum"]) + raw_checksum = structure["checksum"] alg, checksum = raw_checksum.split("$") if alg != SHA1: raise TypeError( @@ -659,7 +661,7 @@ def _relativise_files( # Register in RO; but why was this not picked # up by used_artefacts? _logger.info("[provenance] Adding to RO %s", structure["location"]) - with self.fsaccess.open(cast(str, structure["location"]), "rb") as fp: + with self.fsaccess.open(structure["location"], "rb") as fp: relative_path = self.add_data_file(fp) checksum = PurePosixPath(relative_path).name structure["checksum"] = f"{SHA1}${checksum}" @@ -668,14 +670,14 @@ def _relativise_files( if "path" in structure: del structure["path"] - if structure.get("class") == "Directory": + if is_directory(structure): # TODO: Generate anonymous Directory with a "listing" # pointing to the hashed files del structure["location"] for val in structure.values(): try: - self._relativise_files(val) + self._relativise_files(cast(CWLOutputType, val)) except OSError: pass return diff --git a/cwltool/cwlprov/writablebagfile.py b/cwltool/cwlprov/writablebagfile.py index ecd6463d6..f0f7fb323 100644 --- a/cwltool/cwlprov/writablebagfile.py +++ b/cwltool/cwlprov/writablebagfile.py @@ -14,10 +14,11 @@ from pathlib import Path, PurePosixPath from typing import Any, BinaryIO, cast +from cwl_utils.types import CWLObjectType from schema_salad.utils import json_dumps from ..loghandler import _logger -from ..utils import CWLObjectType, local_path, posix_path +from ..utils import local_path, posix_path from .provenance_constants import ( CWLPROV, CWLPROV_VERSION, diff --git a/cwltool/docker.py b/cwltool/docker.py index 69fecc830..b26ac78c2 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,11 +9,12 @@ import subprocess # nosec import sys import threading -from collections.abc import Callable, MutableMapping +from collections.abc import Callable, MutableMapping, MutableSequence from io import StringIO # pylint: disable=redefined-builtin from typing import Optional, cast import requests +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType from .builder import Builder from .context import RuntimeContext @@ -22,7 +23,7 @@ from .job import ContainerCommandLineJob from .loghandler import _logger from .pathmapper import MapperEnt, PathMapper -from .utils import CWLObjectType, create_tmp_dir, ensure_writable +from .utils import create_tmp_dir, ensure_writable _IMAGES: set[str] = set() _IMAGES_LOCK = threading.Lock() @@ -84,7 +85,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + make_path_mapper: Callable[ + [MutableSequence[CWLFileType | CWLDirectoryType], str, RuntimeContext, bool], PathMapper + ], requirements: list[CWLObjectType], hints: list[CWLObjectType], name: str, @@ -446,7 +449,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + make_path_mapper: Callable[ + [MutableSequence[CWLFileType | CWLDirectoryType], str, RuntimeContext, bool], PathMapper + ], requirements: list[CWLObjectType], hints: list[CWLObjectType], name: str, diff --git a/cwltool/executors.py b/cwltool/executors.py index 491078e83..8bcdfa519 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -13,6 +13,7 @@ from typing import Optional, cast import psutil +from cwl_utils.types import CWLObjectType from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine @@ -27,7 +28,7 @@ from .process import Process, cleanIntermediate, relocateOutputs from .task_queue import TaskQueue from .update import ORIGINAL_CWLVERSION -from .utils import CWLObjectType, JobsType +from .utils import JobsType from .workflow import Workflow from .workflow_job import WorkflowJob, WorkflowJobStep diff --git a/cwltool/factory.py b/cwltool/factory.py index fc00eb061..aee4abb6f 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,12 +1,13 @@ import os from typing import Any +from cwl_utils.types import CWLObjectType + from . import load_tool from .context import LoadingContext, RuntimeContext from .errors import WorkflowException from .executors import JobExecutor, SingleJobExecutor from .process import Process -from .utils import CWLObjectType class WorkflowStatus(Exception): diff --git a/cwltool/job.py b/cwltool/job.py index 4f49c95fe..9def5fddd 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -21,6 +21,7 @@ from typing import IO, TYPE_CHECKING, Optional, TextIO, Union, cast import psutil +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType from prov.model import PROV from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dump, json_dumps @@ -35,9 +36,6 @@ from .process import stage_files from .secrets import SecretStore from .utils import ( - CWLObjectType, - CWLOutputType, - DirectoryType, HasReqsHints, OutputCallbackType, bytes2str_in_dicts, @@ -108,7 +106,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + make_path_mapper: Callable[ + [MutableSequence[CWLFileType | CWLDirectoryType], str, RuntimeContext, bool], PathMapper + ], requirements: list[CWLObjectType], hints: list[CWLObjectType], name: str, @@ -138,11 +138,13 @@ def __init__( self.tmpdir = "" self.environment: MutableMapping[str, str] = {} - self.generatefiles: DirectoryType = { - "class": "Directory", - "listing": [], - "basename": "", - } + self.generatefiles = CWLDirectoryType( + **{ + "class": "Directory", + "listing": [], + "basename": "", + } + ) self.stagedir: str | None = None self.inplace_update = False self.prov_obj: ProvenanceProfile | None = None @@ -160,6 +162,7 @@ def run( self, runtimeContext: RuntimeContext, tmpdir_lock: Union[threading.Lock, None] = None, + # use `threading.Lock | None` when we drop support for Python 3.12 ) -> None: pass diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 853c203be..4c34094c6 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -12,6 +12,7 @@ from typing import Any, Union, cast from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.fetcher import Fetcher @@ -32,7 +33,7 @@ from .loghandler import _logger from .process import Process, get_schema, shortname from .update import ALLUPDATES -from .utils import CWLObjectType, ResolverType, visit_class +from .utils import ResolverType, visit_class docloaderctx: ContextType = { "cwl": "https://w3id.org/cwl/cwl#", diff --git a/cwltool/main.py b/cwltool/main.py index b8d74ded0..582066b05 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -23,6 +23,13 @@ import coloredlogs import requests import ruamel.yaml +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + is_file, +) from rich_argparse import RichHelpFormatter from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML @@ -89,8 +96,6 @@ from .update import ALLUPDATES, UPDATES from .utils import ( DEFAULT_TMP_PREFIX, - CWLObjectType, - CWLOutputType, HasReqsHints, adjustDirObjs, normalizeFilesDirs, @@ -245,8 +250,8 @@ def generate_example_input( for field in cast(list[CWLObjectType], fields): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) - case {"type": str(inp_type), "default": default}: - example = default + case {"type": str(inp_type), "default": default_value}: + example = default_value comment = f"default value of type {inp_type!r}" case {"type": str(inp_type)}: example = defaults.get(inp_type, str(inptype)) @@ -298,7 +303,7 @@ def realize_input_schema( if isinstance(entry["type"], Mapping): entry["type"] = cast( CWLOutputType, - realize_input_schema([entry["type"]], schema_defs), + realize_input_schema([cast(CWLObjectType, entry["type"])], schema_defs), ) if entry["type"] == "array": items = entry["items"] if not isinstance(entry["items"], str) else [entry["items"]] @@ -565,14 +570,14 @@ def prov_deps( document_loader: Loader, uri: str, basedir: str | None = None, -) -> CWLObjectType: +) -> CWLFileType: deps = find_deps(obj, document_loader, uri, basedir=basedir) - def remove_non_cwl(deps: CWLObjectType) -> None: - if "secondaryFiles" in deps: - sec_files = cast(list[CWLObjectType], deps["secondaryFiles"]) + def remove_non_cwl(deps: CWLFileType | CWLDirectoryType) -> None: + if is_file(deps) and "secondaryFiles" in deps: + sec_files = deps["secondaryFiles"] for index, entry in enumerate(sec_files): - if not ("format" in entry and entry["format"] == CWL_IANA): + if not (is_file(entry) and "format" in entry and entry["format"] == CWL_IANA): del sec_files[index] else: remove_non_cwl(entry) @@ -587,13 +592,15 @@ def find_deps( uri: str, basedir: str | None = None, nestdirs: bool = True, -) -> CWLObjectType: +) -> CWLFileType: """Find the dependencies of the CWL document.""" - deps: CWLObjectType = { - "class": "File", - "location": uri, - "format": CWL_IANA, - } + deps = CWLFileType( + **{ + "class": "File", + "location": uri, + "format": CWL_IANA, + } + ) def loadref(base: str, uri: str) -> CommentedMap | CommentedSeq | str | None: return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) @@ -607,7 +614,7 @@ def loadref(base: str, uri: str) -> CommentedMap | CommentedSeq | str | None: nestdirs=nestdirs, ) if sfs is not None: - deps["secondaryFiles"] = cast(MutableSequence[CWLOutputType], mergedirs(sfs)) + deps["secondaryFiles"] = mergedirs(sfs) return deps diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 622807ec6..b3da0a004 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -2,8 +2,9 @@ from typing import NamedTuple, cast +from cwl_utils.types import CWLObjectType + from .errors import WorkflowException -from .utils import CWLObjectType class _MutationState(NamedTuple): diff --git a/cwltool/pack.py b/cwltool/pack.py index 8c250c51b..968e8364c 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -5,6 +5,7 @@ from collections.abc import Callable, MutableMapping, MutableSequence from typing import Any, Optional, Union, cast +from cwl_utils.types import CWLObjectType, CWLOutputType from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader from schema_salad.utils import ResolveType @@ -13,7 +14,6 @@ from .load_tool import fetch_document, resolve_and_validate_document from .process import shortname, uniquename from .update import ORDERED_VERSIONS, ORIGINAL_CWLVERSION, update -from .utils import CWLObjectType, CWLOutputType LoadRefType = Callable[[Optional[str], str], ResolveType] @@ -79,7 +79,7 @@ def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> No def import_embed( - d: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType, + d: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType | None, seen: set[str], ) -> None: if isinstance(d, MutableSequence): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 0cf5f7086..9e25d6c0f 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -3,9 +3,10 @@ import stat import urllib import uuid -from collections.abc import ItemsView, Iterable, Iterator, KeysView +from collections.abc import ItemsView, Iterable, Iterator, KeysView, MutableSequence from typing import NamedTuple, Optional, cast +from cwl_utils.types import CWLDirectoryType, CWLFileType, is_directory from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import uri_file_path @@ -13,7 +14,7 @@ from .loghandler import _logger from .stdfsaccess import abspath -from .utils import CWLObjectType, dedup, downloadHttpFile +from .utils import dedup, downloadHttpFile class MapperEnt(NamedTuple): @@ -75,7 +76,7 @@ class PathMapper: def __init__( self, - referenced_files: list[CWLObjectType], + referenced_files: MutableSequence[CWLFileType | CWLDirectoryType], basedir: str, stagedir: str, separateDirs: bool = True, @@ -88,7 +89,7 @@ def __init__( def visitlisting( self, - listing: list[CWLObjectType], + listing: MutableSequence[CWLFileType | CWLDirectoryType], stagedir: str, basedir: str, copy: bool = False, @@ -105,7 +106,7 @@ def visitlisting( def visit( self, - obj: CWLObjectType, + obj: CWLFileType | CWLDirectoryType, stagedir: str, basedir: str, copy: bool = False, @@ -116,10 +117,10 @@ def visit( stagedir = cast(Optional[str], obj.get("dirname")) or stagedir tgt = os.path.join( stagedir, - cast(str, obj["basename"]), + obj["basename"], ) - if obj["class"] == "Directory": - location = cast(str, obj["location"]) + if is_directory(obj): + location = obj["location"] if location.startswith("file://"): resolved = uri_file_path(location) else: @@ -130,18 +131,18 @@ def visit( if location.startswith("file://"): staged = False self.visitlisting( - cast(list[CWLObjectType], obj.get("listing", [])), + obj.get("listing", []), tgt, basedir, copy=copy, staged=staged, ) - elif obj["class"] == "File": - path = cast(str, obj["location"]) + else: + path = obj["location"] ab = abspath(path, basedir) if "contents" in obj and path.startswith("_:"): self._pathmap[path] = MapperEnt( - cast(str, obj["contents"]), + obj["contents"], tgt, "CreateWritableFile" if copy else "CreateFile", staged, @@ -172,14 +173,16 @@ def visit( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( - cast(list[CWLObjectType], obj.get("secondaryFiles", [])), + obj.get("secondaryFiles", []), stagedir, basedir, copy=copy, staged=staged, ) - def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: + def setup( + self, referenced_files: MutableSequence[CWLFileType | CWLDirectoryType], basedir: str + ) -> None: """ For each file, set the target to its own directory. diff --git a/cwltool/process.py b/cwltool/process.py index 75069b859..4f11b8a24 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -19,6 +19,7 @@ Iterator, MutableMapping, MutableSequence, + Sequence, Sized, ) from importlib.resources import files @@ -26,6 +27,15 @@ from typing import TYPE_CHECKING, Any, Optional, Union, cast from cwl_utils import expression +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + is_directory, + is_file, + is_file_or_directory, +) from mypy_extensions import mypyc_attr from rdflib import Graph from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -52,8 +62,6 @@ from .stdfsaccess import StdFsAccess from .update import INTERNAL_VERSION, ORDERED_VERSIONS, ORIGINAL_CWLVERSION from .utils import ( - CWLObjectType, - CWLOutputType, HasReqsHints, JobsGeneratorType, LoadListingType, @@ -309,9 +317,9 @@ def relocateOutputs( def _collectDirEntries( obj: CWLObjectType | MutableSequence[CWLObjectType] | None, - ) -> Iterator[CWLObjectType]: + ) -> Iterator[CWLFileType | CWLDirectoryType]: if isinstance(obj, dict): - if obj.get("class") in ("File", "Directory"): + if is_file_or_directory(obj): yield obj else: for sub_obj in obj.values(): @@ -447,7 +455,9 @@ def avroize_type( cast(MutableSequence[CWLOutputType], items), name_prefix ) case {"type": f_type}: - cast(CWLObjectType, field_type)["type"] = avroize_type(f_type, name_prefix) + cast(CWLObjectType, field_type)["type"] = avroize_type( + cast(CWLObjectType, f_type), name_prefix + ) case "File": return "org.w3id.cwl.cwl.File" case "Directory": @@ -477,7 +487,7 @@ def get_overrides(overrides: MutableSequence[CWLObjectType], toolid: str) -> CWL def var_spool_cwl_detector( - obj: CWLOutputType, + obj: CWLOutputType | None, item: Any | None = None, obj_key: Any | None = None, ) -> bool: @@ -811,7 +821,7 @@ def inc(d: list[int]) -> None: except (ValidationException, WorkflowException) as err: raise WorkflowException("Invalid job input record:\n" + str(err)) from err - files: list[CWLObjectType] = [] + files: MutableSequence[CWLFileType | CWLDirectoryType] = [] bindings = CommentedSeq() outdir = "" tmpdir = "" @@ -1127,9 +1137,10 @@ def uniquename(stem: str, names: set[str] | None = None) -> str: return u -def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: +def nestdir(base: str, deps: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType: + """Add intermediate directory objects to preserve the relative layout.""" dirname = os.path.dirname(base) + "/" - subid = cast(str, deps["location"]) + subid = deps["location"] if subid.startswith(dirname): s2 = subid[len(dirname) :] sp = s2.split("/") @@ -1137,22 +1148,24 @@ def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: while sp: loc = dirname + "/".join(sp) nx = sp.pop() - deps = { - "class": "Directory", - "basename": nx, - "listing": [deps], - "location": loc, - } + deps = CWLDirectoryType( + **{ + "class": "Directory", + "basename": nx, + "listing": [deps], + "location": loc, + } + ) return deps def mergedirs( - listing: MutableSequence[CWLObjectType], -) -> MutableSequence[CWLObjectType]: - r: list[CWLObjectType] = [] - ents: dict[str, CWLObjectType] = {} + listing: MutableSequence[CWLFileType | CWLDirectoryType], +) -> MutableSequence[CWLFileType | CWLDirectoryType]: + r: list[CWLFileType | CWLDirectoryType] = [] + ents: dict[str, CWLFileType | CWLDirectoryType] = {} for e in listing: - basename = cast(str, e["basename"]) + basename = e["basename"] if basename not in ents: ents[basename] = e elif e["location"] != ents[basename]["location"]: @@ -1160,19 +1173,16 @@ def mergedirs( "Conflicting basename in listing or secondaryFiles, '%s' used by both '%s' and '%s'" % (basename, e["location"], ents[basename]["location"]) ) - elif e["class"] == "Directory": + elif is_directory(e): if e.get("listing"): # name already in entries # merge it into the existing listing - cast(list[CWLObjectType], ents[basename].setdefault("listing", [])).extend( - cast(list[CWLObjectType], e["listing"]) + cast(CWLDirectoryType, ents[basename]).setdefault("listing", []).extend( + e["listing"] ) for e in ents.values(): - if e["class"] == "Directory" and "listing" in e: - e["listing"] = cast( - MutableSequence[CWLOutputType], - mergedirs(cast(list[CWLObjectType], e["listing"])), - ) + if is_directory(e) and "listing" in e: + e["listing"] = mergedirs(e["listing"]) r.extend(ents.values()) return r @@ -1182,13 +1192,18 @@ def mergedirs( def scandeps( base: str, - doc: CWLObjectType | MutableSequence[CWLObjectType], + doc: ( + CWLFileType + | CWLDirectoryType + | CWLObjectType + | Sequence[CWLFileType | CWLDirectoryType | CWLObjectType] + ), reffields: set[str], urlfields: set[str], loadref: Callable[[str, str], CommentedMap | CommentedSeq | str | None], urljoin: Callable[[str, str], str] = urllib.parse.urljoin, nestdirs: bool = True, -) -> MutableSequence[CWLObjectType]: +) -> MutableSequence[CWLFileType | CWLDirectoryType]: """ Search for external files references in a CWL document or input object. @@ -1208,72 +1223,74 @@ def scandeps( produce the same relative file system locations. :returns: A list of File or Directory dependencies """ - r: MutableSequence[CWLObjectType] = [] + r: MutableSequence[CWLFileType | CWLDirectoryType] = [] if isinstance(doc, MutableMapping): if "id" in doc: if cast(str, doc["id"]).startswith("file://"): df, _ = urllib.parse.urldefrag(cast(str, doc["id"])) if base != df: - r.append({"class": "File", "location": df, "format": CWL_IANA}) + r.append(CWLFileType(**{"class": "File", "location": df, "format": CWL_IANA})) base = df - if doc.get("class") in ("File", "Directory") and "location" in urlfields: - u = cast(Optional[str], doc.get("location", doc.get("path"))) + if is_file_or_directory(doc) and "location" in urlfields: + u = doc.get("location", doc.get("path")) if u and not u.startswith("_:"): - deps: CWLObjectType = { - "class": doc["class"], - "location": urljoin(base, u), - } + deps: CWLFileType | CWLDirectoryType + if is_file(doc): + deps = CWLFileType( + **{ + "class": "File", + } + ) + if "secondaryFiles" in doc: + deps["secondaryFiles"] = scandeps( + base, + doc["secondaryFiles"], + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, + ) + else: + deps = CWLDirectoryType( + **{ + "class": "Directory", + } + ) + if "listing" in doc: + deps["listing"] = doc["listing"] + deps["location"] = urljoin(base, u) if "basename" in doc: deps["basename"] = doc["basename"] - match doc: - case {"class": "Directory", "listing": listing}: - deps["listing"] = listing - case {"class": "File", "secondaryFiles": sec_files}: - deps["secondaryFiles"] = cast( - CWLOutputType, - scandeps( - base, - cast( - Union[CWLObjectType, MutableSequence[CWLObjectType]], - sec_files, - ), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, - ), - ) if nestdirs: deps = nestdir(base, deps) r.append(deps) else: - match doc: - case {"class": "Directory", "listing": listing}: - r.extend( - scandeps( - base, - cast(MutableSequence[CWLObjectType], listing), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, - ) + if is_directory(doc) and "listing" in doc: + r.extend( + scandeps( + base, + doc["listing"], + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, ) - case {"class": "File", "secondaryFiles": sec_files}: - r.extend( - scandeps( - base, - cast(MutableSequence[CWLObjectType], sec_files), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, - ) + ) + elif is_file(doc) and "secondaryFiles" in doc: + r.extend( + scandeps( + base, + doc["secondaryFiles"], + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, ) + ) for k, v in doc.items(): if k in reffields: @@ -1300,11 +1317,13 @@ def scandeps( Union[MutableSequence[CWLObjectType], CWLObjectType], loadref(base, u2), ) - deps2: CWLObjectType = { - "class": "File", - "location": subid, - "format": CWL_IANA, - } + deps2 = CWLFileType( + **{ + "class": "File", + "location": subid, + "format": CWL_IANA, + } + ) sf = scandeps( subid, sub, @@ -1315,19 +1334,18 @@ def scandeps( nestdirs=nestdirs, ) if sf: - deps2["secondaryFiles"] = cast( - MutableSequence[CWLOutputType], mergedirs(sf) - ) + deps2["secondaryFiles"] = mergedirs(sf) if nestdirs: - deps2 = nestdir(base, deps2) - r.append(deps2) + r.append(nestdir(base, deps2)) + else: + r.append(deps2) elif k in urlfields and k != "location": for u3 in aslist(v): - deps = {"class": "File", "location": urljoin(base, u3)} + deps = CWLFileType(**{"class": "File", "location": urljoin(base, u3)}) if nestdirs: deps = nestdir(base, deps) r.append(deps) - elif doc.get("class") in ("File", "Directory") and k in ( + elif is_file_or_directory(doc) and k in ( "listing", "secondaryFiles", ): diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index eb7eca076..1dd963b06 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,6 +1,7 @@ import copy from typing import cast +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException from schema_salad.sourceline import indent @@ -10,7 +11,7 @@ from .load_tool import load_tool from .loghandler import _logger from .process import Process, shortname -from .utils import CWLObjectType, JobsGeneratorType, OutputCallbackType +from .utils import JobsGeneratorType, OutputCallbackType class ProcessGeneratorJob: diff --git a/cwltool/secrets.py b/cwltool/secrets.py index 6a39231e4..4fea75994 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -3,7 +3,7 @@ import uuid from collections.abc import MutableMapping, MutableSequence -from .utils import CWLObjectType, CWLOutputType +from cwl_utils.types import CWLObjectType, CWLOutputType class SecretStore: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 5551882b8..75ebb769d 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -9,10 +9,11 @@ import shutil import sys import threading -from collections.abc import Callable, MutableMapping +from collections.abc import Callable, MutableMapping, MutableSequence from subprocess import check_call, check_output # nosec from typing import cast +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType from packaging.version import Version from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dumps @@ -28,7 +29,7 @@ from .loghandler import _logger from .pathmapper import MapperEnt, PathMapper from .singularity_utils import singularity_supports_userns -from .utils import CWLObjectType, create_tmp_dir, ensure_non_writable, ensure_writable +from .utils import create_tmp_dir, ensure_non_writable, ensure_writable # Cached version number of singularity # This is a list containing major and minor versions as integer. @@ -169,7 +170,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + make_path_mapper: Callable[ + [MutableSequence[CWLFileType | CWLDirectoryType], str, RuntimeContext, bool], PathMapper + ], requirements: list[CWLObjectType], hints: list[CWLObjectType], name: str, diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 8bad72a47..5378ec3ad 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -2,11 +2,12 @@ from collections.abc import Mapping, MutableMapping, MutableSequence from typing import Any, NamedTuple, Union, cast +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap, CommentedSeq from .context import LoadingContext from .load_tool import load_tool, make_tool -from .utils import CWLObjectType, aslist +from .utils import aslist from .workflow import Workflow, WorkflowStep diff --git a/cwltool/update.py b/cwltool/update.py index 91a63f496..48b9484c9 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -3,13 +3,14 @@ from functools import partial from typing import cast +from cwl_utils.types import CWLObjectType, CWLOutputType from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader from schema_salad.sourceline import SourceLine from .loghandler import _logger -from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field +from .utils import aslist, visit_class, visit_field def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> tuple[CommentedMap, str]: @@ -124,7 +125,7 @@ def rewrite_requirements(t: CWLObjectType) -> None: rewrite_requirements(s) def update_secondaryFiles( - t: CWLOutputType, top: bool = False + t: CWLOutputType | None, top: bool = False ) -> MutableSequence[MutableMapping[str, str]] | MutableMapping[str, str]: if isinstance(t, CommentedSeq): new_seq = copy.deepcopy(t) diff --git a/cwltool/utils.py b/cwltool/utils.py index 2975f0199..f3702192c 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -2,6 +2,15 @@ import collections +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + is_directory, + is_file, +) + try: import fcntl except ImportError: @@ -25,6 +34,7 @@ Iterable, MutableMapping, MutableSequence, + Sequence, ) from datetime import datetime from email.utils import parsedate_to_datetime @@ -41,9 +51,7 @@ NamedTuple, Optional, TypeAlias, - TypedDict, Union, - cast, ) import requests @@ -68,16 +76,6 @@ processes_to_kill: Deque["subprocess.Popen[str]"] = collections.deque() -CWLOutputType: TypeAlias = Union[ - None, - bool, - str, - int, - float, - MutableSequence["CWLOutputType"], - MutableMapping[str, "CWLOutputType"], -] -CWLObjectType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]] """Typical raw dictionary found in lightly parsed CWL.""" JobsType: TypeAlias = Union[ @@ -89,10 +87,6 @@ DestinationsType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]] ScatterDestinationsType: TypeAlias = MutableMapping[str, list[Optional[CWLOutputType]]] ScatterOutputCallbackType: TypeAlias = Callable[[Optional[ScatterDestinationsType], str], None] -SinkType: TypeAlias = Union[CWLOutputType, CWLObjectType] -DirectoryType = TypedDict( - "DirectoryType", {"class": str, "listing": list[CWLObjectType], "basename": str} -) JSONType: TypeAlias = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] @@ -254,15 +248,18 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No visit_class(rec, ("Directory",), op) -def dedup(listing: list[CWLObjectType]) -> list[CWLObjectType]: +def dedup( + listing: MutableSequence[CWLFileType | CWLDirectoryType], +) -> MutableSequence[CWLFileType | CWLDirectoryType]: + """Remove duplicate entries from a CWL Directory 'listing'.""" marksub = set() def mark(d: dict[str, str]) -> None: marksub.add(d["location"]) for entry in listing: - if entry["class"] == "Directory": - for e in cast(list[CWLObjectType], entry.get("listing", [])): + if is_directory(entry): + for e in entry.get("listing", []): adjustFileObjs(e, mark) adjustDirObjs(e, mark) @@ -271,37 +268,41 @@ def mark(d: dict[str, str]) -> None: for r in listing: if r["location"] not in marksub and r["location"] not in markdup: dd.append(r) - markdup.add(cast(str, r["location"])) + markdup.add(r["location"]) return dd -def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = True) -> None: +def get_listing( + fs_access: "StdFsAccess", rec: CWLObjectType | CWLDirectoryType, recursive: bool = True +) -> None: """Expand, recursively, any 'listing' fields in a Directory.""" - if rec.get("class") != "Directory": - finddirs: list[CWLObjectType] = [] + if not is_directory(rec): + finddirs: list[CWLDirectoryType] = [] visit_class(rec, ("Directory",), finddirs.append) for f in finddirs: get_listing(fs_access, f, recursive=recursive) return if "listing" in rec: return - listing: list[CWLOutputType] = [] - loc = cast(str, rec["location"]) + listing: MutableSequence[CWLFileType | CWLDirectoryType] = [] + loc = rec["location"] for ld in fs_access.listdir(loc): parse = urllib.parse.urlparse(ld) bn = os.path.basename(urllib.request.url2pathname(parse.path)) if fs_access.isdir(ld): - ent: MutableMapping[str, Any] = { - "class": "Directory", - "location": ld, - "basename": bn, - } + ent = CWLDirectoryType( + **{ + "class": "Directory", + "location": ld, + "basename": bn, + } + ) if recursive: get_listing(fs_access, ent, recursive) listing.append(ent) else: - listing.append({"class": "File", "location": ld, "basename": bn}) + listing.append(CWLFileType(**{"class": "File", "location": ld, "basename": bn})) rec["listing"] = listing @@ -401,17 +402,24 @@ def ensure_non_writable(path: str) -> None: def normalizeFilesDirs( - job: None | ( - MutableSequence[MutableMapping[str, Any]] | MutableMapping[str, Any] | DirectoryType - ), + job: Sequence[CWLObjectType | CWLOutputType | None] | CWLObjectType | CWLOutputType | None, ) -> None: - def addLocation(d: dict[str, Any]) -> None: + """ + Add missing `location`s and `basename`s to CWL File and Directory objects. + + :raises ValidationException: if anonymous objects are missing required fields, + or if the location ends in '/' but the object isn't + a directory + + """ + + def addLocation(d: CWLFileType | CWLDirectoryType) -> None: if "location" not in d: - if d["class"] == "File" and ("contents" not in d): + if is_file(d) and ("contents" not in d): raise ValidationException( "Anonymous file object must have 'contents' and 'basename' fields." ) - if d["class"] == "Directory" and ("listing" not in d or "basename" not in d): + if is_directory(d) and ("listing" not in d or "basename" not in d): raise ValidationException( "Anonymous directory object must have 'listing' and 'basename' fields." ) @@ -423,7 +431,7 @@ def addLocation(d: dict[str, Any]) -> None: path = parse.path # strip trailing slash if path.endswith("/"): - if d["class"] != "Directory": + if not is_directory(d): raise ValidationException( "location '%s' ends with '/' but is not a Directory" % d["location"] ) @@ -445,7 +453,7 @@ def addLocation(d: dict[str, Any]) -> None: else: d["basename"] = str(os.path.basename(urllib.request.url2pathname(path))) - if d["class"] == "File": + if is_file(d): nr, ne = os.path.splitext(d["basename"]) if d.get("nameroot") != nr: d["nameroot"] = str(nr) diff --git a/cwltool/workflow.py b/cwltool/workflow.py index fe34c75ce..6a66b3ef2 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -7,6 +7,7 @@ from typing import cast from uuid import UUID +from cwl_utils.types import CWLObjectType from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException @@ -21,13 +22,7 @@ from .load_tool import load_tool from .loghandler import _logger from .process import Process, get_overrides, shortname -from .utils import ( - CWLObjectType, - JobsGeneratorType, - OutputCallbackType, - StepType, - aslist, -) +from .utils import JobsGeneratorType, OutputCallbackType, StepType, aslist from .workflow_job import WorkflowJob diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 6e1527c20..c1a3f7d2d 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Optional, Union, cast from cwl_utils import expression +from cwl_utils.types import CWLObjectType, CWLOutputType, SinkType from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dumps @@ -18,14 +19,11 @@ from .process import shortname, uniquename from .stdfsaccess import StdFsAccess from .utils import ( - CWLObjectType, - CWLOutputType, JobsGeneratorType, OutputCallbackType, ParametersType, ScatterDestinationsType, ScatterOutputCallbackType, - SinkType, WorkflowStateItem, adjustDirObjs, aslist, diff --git a/mypy-requirements.txt b/mypy-requirements.txt index e2dd3606d..c6c3bb105 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,6 +1,5 @@ mypy==1.19.1 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 -cwl-utils>=0.32 cwltest types-requests types-setuptools diff --git a/pyproject.toml b/pyproject.toml index 57b89aa93..81d17192c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "types-psutil>=7.1.3.20251210", "ruamel.yaml>=0.16.0,<0.19", "schema-salad>=8.9,<9", - "cwl-utils>=0.32", + "cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head", "toml", "argcomplete>=1.12.0", "rich-argparse", diff --git a/requirements.txt b/requirements.txt index 6dcb90bb9..0ccb07436 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ coloredlogs pydot>=1.4.1 argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 -cwl-utils>=0.32 +cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head spython>=0.3.0 rich-argparse typing-extensions>=4.1.0 diff --git a/setup.py b/setup.py index e473c36fe..ba3c2c8d0 100644 --- a/setup.py +++ b/setup.py @@ -159,7 +159,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li "pydot >= 1.4.1", "argcomplete >= 1.12.0", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 - "cwl-utils >= 0.32", + "cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head", "spython >= 0.3.0", "rich-argparse", "typing-extensions >= 4.1.0", diff --git a/tests/test_context.py b/tests/test_context.py index 505dfd635..b417180c4 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -6,9 +6,10 @@ from pathlib import Path from typing import cast +from cwl_utils.types import CWLObjectType + from cwltool.context import RuntimeContext from cwltool.factory import Factory -from cwltool.utils import CWLObjectType from cwltool.workflow_job import WorkflowJobStep from .util import get_data, needs_docker diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 93b1c7a47..391d3fa65 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock import pytest +from cwl_utils.types import CWLObjectType from schema_salad.avro import schema from cwltool.builder import Builder @@ -15,7 +16,6 @@ from cwltool.process import use_custom_schema from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION -from cwltool.utils import CWLObjectType from .util import get_data, needs_docker, needs_singularity_3_or_newer diff --git a/tests/test_examples.py b/tests/test_examples.py index f371f45ae..329bb6d47 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -7,6 +7,7 @@ import subprocess import sys import urllib.parse +from collections.abc import MutableMapping, MutableSequence from io import StringIO from pathlib import Path from typing import Any, cast @@ -16,6 +17,13 @@ import pytest from cwl_utils.errors import JavascriptException from cwl_utils.sandboxjs import param_re +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + CWLParameterContext, +) from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -29,37 +37,37 @@ from cwltool.errors import WorkflowException from cwltool.main import main from cwltool.process import CWL_IANA -from cwltool.utils import CWLObjectType, dedup +from cwltool.utils import dedup from .util import get_data, get_main_output, needs_docker, working_directory sys.argv = [""] expression_match = [ - ("(foo)", True), - ("(foo.bar)", True), - ("(foo['bar'])", True), - ('(foo["bar"])', True), - ("(foo.bar.baz)", True), - ("(foo['bar'].baz)", True), - ("(foo['bar']['baz'])", True), - ("(foo['b\\'ar']['baz'])", True), - ("(foo['b ar']['baz'])", True), - ("(foo_bar)", True), - ('(foo.["bar"])', False), - ('(.foo["bar"])', False), - ('(foo ["bar"])', False), - ('( foo["bar"])', False), - ("(foo[bar].baz)", False), - ("(foo['bar\"].baz)", False), - ("(foo['bar].baz)", False), - ("{foo}", False), - ("(foo.bar", False), - ("foo.bar)", False), - ("foo.b ar)", False), - ("foo.b'ar)", False), - ("(foo+bar", False), - ("(foo bar", False), + ("(inputs)", True), + ("(inputs.bar)", True), + ("(inputs['bar'])", True), + ('(inputs["bar"])', True), + ("(inputs.bar.baz)", True), + ("(inputs['bar'].baz)", True), + ("(inputs['bar']['baz'])", True), + ("(inputs['b\\'ar']['baz'])", True), + ("(inputs['b ar']['baz'])", True), + ("(inputs_bar)", True), + ('(inputs.["bar"])', False), + ('(.inputs["bar"])', False), + ('(inputs ["bar"])', False), + ('( inputs["bar"])', False), + ("(inputs[bar].baz)", False), + ("(inputs['bar\"].baz)", False), + ("(inputs['bar].baz)", False), + ("{inputs}", False), + ("(inputs.bar", False), + ("inputs.bar)", False), + ("inputs.b ar)", False), + ("inputs.b'ar)", False), + ("(inputs+bar", False), + ("(inputs bar", False), ] @@ -69,54 +77,66 @@ def test_expression_match(expression: str, expected: bool) -> None: assert (match is not None) == expected -interpolate_input: dict[str, Any] = { - "foo": { +interpolate_input = CWLParameterContext( + inputs={ "bar": {"baz": "zab1"}, "b ar": {"baz": 2}, "b'ar": {"baz": True}, 'b"ar': {"baz": None}, }, - "lst": ["A", "B"], -} + self=["A", "B"], +) interpolate_parameters = [ - ("$(foo)", interpolate_input["foo"]), - ("$(foo.bar)", interpolate_input["foo"]["bar"]), - ("$(foo['bar'])", interpolate_input["foo"]["bar"]), - ('$(foo["bar"])', interpolate_input["foo"]["bar"]), - ("$(foo.bar.baz)", interpolate_input["foo"]["bar"]["baz"]), - ("$(foo['bar'].baz)", interpolate_input["foo"]["bar"]["baz"]), - ("$(foo['bar'][\"baz\"])", interpolate_input["foo"]["bar"]["baz"]), - ("$(foo.bar['baz'])", interpolate_input["foo"]["bar"]["baz"]), - ("$(foo['b\\'ar'].baz)", True), - ('$(foo["b\'ar"].baz)', True), - ("$(foo['b\\\"ar'].baz)", None), - ("$(lst[0])", "A"), - ("$(lst[1])", "B"), - ("$(lst.length)", 2), - ("$(lst['length'])", 2), - ("-$(foo.bar)", """-{"baz": "zab1"}"""), - ("-$(foo['bar'])", """-{"baz": "zab1"}"""), - ('-$(foo["bar"])', """-{"baz": "zab1"}"""), - ("-$(foo.bar.baz)", "-zab1"), - ("-$(foo['bar'].baz)", "-zab1"), - ("-$(foo['bar'][\"baz\"])", "-zab1"), - ("-$(foo.bar['baz'])", "-zab1"), - ("-$(foo['b ar'].baz)", "-2"), - ("-$(foo['b\\'ar'].baz)", "-true"), - ('-$(foo["b\\\'ar"].baz)', "-true"), - ("-$(foo['b\\\"ar'].baz)", "-null"), - ("$(foo.bar) $(foo.bar)", """{"baz": "zab1"} {"baz": "zab1"}"""), - ("$(foo['bar']) $(foo['bar'])", """{"baz": "zab1"} {"baz": "zab1"}"""), - ('$(foo["bar"]) $(foo["bar"])', """{"baz": "zab1"} {"baz": "zab1"}"""), - ("$(foo.bar.baz) $(foo.bar.baz)", "zab1 zab1"), - ("$(foo['bar'].baz) $(foo['bar'].baz)", "zab1 zab1"), - ("$(foo['bar'][\"baz\"]) $(foo['bar'][\"baz\"])", "zab1 zab1"), - ("$(foo.bar['baz']) $(foo.bar['baz'])", "zab1 zab1"), - ("$(foo['b ar'].baz) $(foo['b ar'].baz)", "2 2"), - ("$(foo['b\\'ar'].baz) $(foo['b\\'ar'].baz)", "true true"), - ('$(foo["b\\\'ar"].baz) $(foo["b\\\'ar"].baz)', "true true"), - ("$(foo['b\\\"ar'].baz) $(foo['b\\\"ar'].baz)", "null null"), + ("$(inputs)", interpolate_input["inputs"]), + ("$(inputs.bar)", interpolate_input["inputs"]["bar"]), + ("$(inputs['bar'])", interpolate_input["inputs"]["bar"]), + ('$(inputs["bar"])', interpolate_input["inputs"]["bar"]), + ( + "$(inputs.bar.baz)", + cast(MutableMapping[str, CWLOutputType], interpolate_input["inputs"]["bar"])["baz"], + ), + ( + "$(inputs['bar'].baz)", + cast(MutableMapping[str, CWLOutputType], interpolate_input["inputs"]["bar"])["baz"], + ), + ( + "$(inputs['bar'][\"baz\"])", + cast(MutableMapping[str, CWLOutputType], interpolate_input["inputs"]["bar"])["baz"], + ), + ( + "$(inputs.bar['baz'])", + cast(MutableMapping[str, CWLOutputType], interpolate_input["inputs"]["bar"])["baz"], + ), + ("$(inputs['b\\'ar'].baz)", True), + ('$(inputs["b\'ar"].baz)', True), + ("$(inputs['b\\\"ar'].baz)", None), + ("$(self[0])", "A"), + ("$(self[1])", "B"), + ("$(self.length)", 2), + ("$(self['length'])", 2), + ("-$(inputs.bar)", """-{"baz": "zab1"}"""), + ("-$(inputs['bar'])", """-{"baz": "zab1"}"""), + ('-$(inputs["bar"])', """-{"baz": "zab1"}"""), + ("-$(inputs.bar.baz)", "-zab1"), + ("-$(inputs['bar'].baz)", "-zab1"), + ("-$(inputs['bar'][\"baz\"])", "-zab1"), + ("-$(inputs.bar['baz'])", "-zab1"), + ("-$(inputs['b ar'].baz)", "-2"), + ("-$(inputs['b\\'ar'].baz)", "-true"), + ('-$(inputs["b\\\'ar"].baz)', "-true"), + ("-$(inputs['b\\\"ar'].baz)", "-null"), + ("$(inputs.bar) $(inputs.bar)", """{"baz": "zab1"} {"baz": "zab1"}"""), + ("$(inputs['bar']) $(inputs['bar'])", """{"baz": "zab1"} {"baz": "zab1"}"""), + ('$(inputs["bar"]) $(inputs["bar"])', """{"baz": "zab1"} {"baz": "zab1"}"""), + ("$(inputs.bar.baz) $(inputs.bar.baz)", "zab1 zab1"), + ("$(inputs['bar'].baz) $(inputs['bar'].baz)", "zab1 zab1"), + ("$(inputs['bar'][\"baz\"]) $(inputs['bar'][\"baz\"])", "zab1 zab1"), + ("$(inputs.bar['baz']) $(inputs.bar['baz'])", "zab1 zab1"), + ("$(inputs['b ar'].baz) $(inputs['b ar'].baz)", "2 2"), + ("$(inputs['b\\'ar'].baz) $(inputs['b\\'ar'].baz)", "true true"), + ('$(inputs["b\\\'ar"].baz) $(inputs["b\\\'ar"].baz)', "true true"), + ("$(inputs['b\\\"ar'].baz) $(inputs['b\\\"ar'].baz)", "null null"), ] @@ -127,31 +147,31 @@ def test_expression_interpolate(pattern: str, expected: Any) -> None: parameter_to_expressions = [ ( - "-$(foo)", + "-$(inputs)", r"""-{"bar":{"baz":"zab1"},"b ar":{"baz":2},"b'ar":{"baz":true},"b\"ar":{"baz":null}}""", ), - ("-$(foo.bar)", """-{"baz":"zab1"}"""), - ("-$(foo['bar'])", """-{"baz":"zab1"}"""), - ('-$(foo["bar"])', """-{"baz":"zab1"}"""), - ("-$(foo.bar.baz)", "-zab1"), - ("-$(foo['bar'].baz)", "-zab1"), - ("-$(foo['bar'][\"baz\"])", "-zab1"), - ("-$(foo.bar['baz'])", "-zab1"), - ("-$(foo['b ar'].baz)", "-2"), - ("-$(foo['b\\'ar'].baz)", "-true"), - ('-$(foo["b\\\'ar"].baz)', "-true"), - ("-$(foo['b\\\"ar'].baz)", "-null"), - ("$(foo.bar) $(foo.bar)", """{"baz":"zab1"} {"baz":"zab1"}"""), - ("$(foo['bar']) $(foo['bar'])", """{"baz":"zab1"} {"baz":"zab1"}"""), - ('$(foo["bar"]) $(foo["bar"])', """{"baz":"zab1"} {"baz":"zab1"}"""), - ("$(foo.bar.baz) $(foo.bar.baz)", "zab1 zab1"), - ("$(foo['bar'].baz) $(foo['bar'].baz)", "zab1 zab1"), - ("$(foo['bar'][\"baz\"]) $(foo['bar'][\"baz\"])", "zab1 zab1"), - ("$(foo.bar['baz']) $(foo.bar['baz'])", "zab1 zab1"), - ("$(foo['b ar'].baz) $(foo['b ar'].baz)", "2 2"), - ("$(foo['b\\'ar'].baz) $(foo['b\\'ar'].baz)", "true true"), - ('$(foo["b\\\'ar"].baz) $(foo["b\\\'ar"].baz)', "true true"), - ("$(foo['b\\\"ar'].baz) $(foo['b\\\"ar'].baz)", "null null"), + ("-$(inputs.bar)", """-{"baz":"zab1"}"""), + ("-$(inputs['bar'])", """-{"baz":"zab1"}"""), + ('-$(inputs["bar"])', """-{"baz":"zab1"}"""), + ("-$(inputs.bar.baz)", "-zab1"), + ("-$(inputs['bar'].baz)", "-zab1"), + ("-$(inputs['bar'][\"baz\"])", "-zab1"), + ("-$(inputs.bar['baz'])", "-zab1"), + ("-$(inputs['b ar'].baz)", "-2"), + ("-$(inputs['b\\'ar'].baz)", "-true"), + ('-$(inputs["b\\\'ar"].baz)', "-true"), + ("-$(inputs['b\\\"ar'].baz)", "-null"), + ("$(inputs.bar) $(inputs.bar)", """{"baz":"zab1"} {"baz":"zab1"}"""), + ("$(inputs['bar']) $(inputs['bar'])", """{"baz":"zab1"} {"baz":"zab1"}"""), + ('$(inputs["bar"]) $(inputs["bar"])', """{"baz":"zab1"} {"baz":"zab1"}"""), + ("$(inputs.bar.baz) $(inputs.bar.baz)", "zab1 zab1"), + ("$(inputs['bar'].baz) $(inputs['bar'].baz)", "zab1 zab1"), + ("$(inputs['bar'][\"baz\"]) $(inputs['bar'][\"baz\"])", "zab1 zab1"), + ("$(inputs.bar['baz']) $(inputs.bar['baz'])", "zab1 zab1"), + ("$(inputs['b ar'].baz) $(inputs['b ar'].baz)", "2 2"), + ("$(inputs['b\\'ar'].baz) $(inputs['b\\'ar'].baz)", "true true"), + ('$(inputs["b\\\'ar"].baz) $(inputs["b\\\'ar"].baz)', "true true"), + ("$(inputs['b\\\"ar'].baz) $(inputs['b\\\"ar'].baz)", "null null"), ] @@ -173,22 +193,22 @@ def test_parameter_to_expression(pattern: str, expected: Any) -> None: param_to_expr_interpolate_escapebehavior = ( - ("\\$(foo.bar.baz)", "$(foo.bar.baz)", 1), - ("\\\\$(foo.bar.baz)", "\\zab1", 1), - ("\\\\\\$(foo.bar.baz)", "\\$(foo.bar.baz)", 1), - ("\\\\\\\\$(foo.bar.baz)", "\\\\zab1", 1), - ("\\$foo", "$foo", 1), - ("\\foo", "foo", 1), + ("\\$(inputs.bar.baz)", "$(inputs.bar.baz)", 1), + ("\\\\$(inputs.bar.baz)", "\\zab1", 1), + ("\\\\\\$(inputs.bar.baz)", "\\$(inputs.bar.baz)", 1), + ("\\\\\\\\$(inputs.bar.baz)", "\\\\zab1", 1), + ("\\$inputs", "$inputs", 1), + ("\\inputs", "inputs", 1), ("\\x", "x", 1), ("\\\\x", "\\x", 1), ("\\\\\\x", "\\x", 1), ("\\\\\\\\x", "\\\\x", 1), - ("\\$(foo.bar.baz)", "$(foo.bar.baz)", 2), - ("\\\\$(foo.bar.baz)", "\\zab1", 2), - ("\\\\\\$(foo.bar.baz)", "\\$(foo.bar.baz)", 2), - ("\\\\\\\\$(foo.bar.baz)", "\\\\zab1", 2), - ("\\$foo", "\\$foo", 2), - ("\\foo", "\\foo", 2), + ("\\$(inputs.bar.baz)", "$(inputs.bar.baz)", 2), + ("\\\\$(inputs.bar.baz)", "\\zab1", 2), + ("\\\\\\$(inputs.bar.baz)", "\\$(inputs.bar.baz)", 2), + ("\\\\\\\\$(inputs.bar.baz)", "\\\\zab1", 2), + ("\\$inputs", "\\$inputs", 2), + ("\\inputs", "\\inputs", 2), ("\\x", "\\x", 2), ("\\\\x", "\\x", 2), ("\\\\\\x", "\\\\x", 2), @@ -218,32 +238,32 @@ def test_parameter_to_expression_interpolate_escapebehavior( interpolate_bad_parameters = [ - ("$(fooz)"), - ("$(foo.barz)"), - ("$(foo['barz'])"), - ('$(foo["barz"])'), - ("$(foo.bar.bazz)"), - ("$(foo['bar'].bazz)"), - ("$(foo['bar'][\"bazz\"])"), - ("$(foo.bar['bazz'])"), - ("$(foo['b\\'ar'].bazz)"), - ('$(foo["b\'ar"].bazz)'), - ("$(foo['b\\\"ar'].bazz)"), - ("$(lst[O])"), # not "0" the number, but the letter O - ("$(lst[2])"), - ("$(lst.lengthz)"), - ("$(lst['lengthz'])"), - ("-$(foo.barz)"), - ("-$(foo['barz'])"), - ('-$(foo["barz"])'), - ("-$(foo.bar.bazz)"), - ("-$(foo['bar'].bazz)"), - ("-$(foo['bar'][\"bazz\"])"), - ("-$(foo.bar['bazz'])"), - ("-$(foo['b ar'].bazz)"), - ("-$(foo['b\\'ar'].bazz)"), - ('-$(foo["b\\\'ar"].bazz)'), - ("-$(foo['b\\\"ar'].bazz)"), + ("$(inputsz)"), + ("$(inputs.barz)"), + ("$(inputs['barz'])"), + ('$(inputs["barz"])'), + ("$(inputs.bar.bazz)"), + ("$(inputs['bar'].bazz)"), + ("$(inputs['bar'][\"bazz\"])"), + ("$(inputs.bar['bazz'])"), + ("$(inputs['b\\'ar'].bazz)"), + ('$(inputs["b\'ar"].bazz)'), + ("$(inputs['b\\\"ar'].bazz)"), + ("$(self[O])"), # not "0" the number, but the letter O + ("$(self[2])"), + ("$(self.lengthz)"), + ("$(self['lengthz'])"), + ("-$(inputs.barz)"), + ("-$(inputs['barz'])"), + ('-$(inputs["barz"])'), + ("-$(inputs.bar.bazz)"), + ("-$(inputs['bar'].bazz)"), + ("-$(inputs['bar'][\"bazz\"])"), + ("-$(inputs.bar['bazz'])"), + ("-$(inputs['b ar'].bazz)"), + ("-$(inputs['b\\'ar'].bazz)"), + ('-$(inputs["b\\\'ar"].bazz)'), + ("-$(inputs['b\\\"ar'].bazz)"), ] @@ -254,22 +274,22 @@ def test_expression_interpolate_failures(pattern: str) -> None: interpolate_escapebehavior = ( - ("\\$(foo.bar.baz)", "$(foo.bar.baz)", 1), - ("\\\\$(foo.bar.baz)", "\\zab1", 1), - ("\\\\\\$(foo.bar.baz)", "\\$(foo.bar.baz)", 1), - ("\\\\\\\\$(foo.bar.baz)", "\\\\zab1", 1), - ("\\$foo", "$foo", 1), - ("\\foo", "foo", 1), + ("\\$(inputs.bar.baz)", "$(inputs.bar.baz)", 1), + ("\\\\$(inputs.bar.baz)", "\\zab1", 1), + ("\\\\\\$(inputs.bar.baz)", "\\$(inputs.bar.baz)", 1), + ("\\\\\\\\$(inputs.bar.baz)", "\\\\zab1", 1), + ("\\$inputs", "$inputs", 1), + ("\\inputs", "inputs", 1), ("\\x", "x", 1), ("\\\\x", "\\x", 1), ("\\\\\\x", "\\x", 1), ("\\\\\\\\x", "\\\\x", 1), - ("\\$(foo.bar.baz)", "$(foo.bar.baz)", 2), - ("\\\\$(foo.bar.baz)", "\\zab1", 2), - ("\\\\\\$(foo.bar.baz)", "\\$(foo.bar.baz)", 2), - ("\\\\\\\\$(foo.bar.baz)", "\\\\zab1", 2), - ("\\$foo", "\\$foo", 2), - ("\\foo", "\\foo", 2), + ("\\$(inputs.bar.baz)", "$(inputs.bar.baz)", 2), + ("\\\\$(inputs.bar.baz)", "\\zab1", 2), + ("\\\\\\$(inputs.bar.baz)", "\\$(inputs.bar.baz)", 2), + ("\\\\\\\\$(inputs.bar.baz)", "\\\\zab1", 2), + ("\\$inputs", "\\$inputs", 2), + ("\\inputs", "\\inputs", 2), ("\\x", "\\x", 2), ("\\\\x", "\\x", 2), ("\\\\\\x", "\\\\x", 2), @@ -576,24 +596,28 @@ def test_scandeps_defaults_with_secondaryfiles() -> None: def test_dedupe() -> None: - not_deduped: list[CWLObjectType] = [ - {"class": "File", "location": "file:///example/a"}, - {"class": "File", "location": "file:///example/a"}, - {"class": "File", "location": "file:///example/d"}, - { - "class": "Directory", - "location": "file:///example/c", - "listing": [{"class": "File", "location": "file:///example/d"}], - }, + not_deduped: MutableSequence[CWLFileType | CWLDirectoryType] = [ + CWLFileType(**{"class": "File", "location": "file:///example/a"}), + CWLFileType(**{"class": "File", "location": "file:///example/a"}), + CWLFileType(**{"class": "File", "location": "file:///example/d"}), + CWLDirectoryType( + **{ + "class": "Directory", + "location": "file:///example/c", + "listing": [{"class": "File", "location": "file:///example/d"}], + } + ), ] expected = [ - {"class": "File", "location": "file:///example/a"}, - { - "class": "Directory", - "location": "file:///example/c", - "listing": [{"class": "File", "location": "file:///example/d"}], - }, + CWLFileType(**{"class": "File", "location": "file:///example/a"}), + CWLDirectoryType( + **{ + "class": "Directory", + "location": "file:///example/c", + "listing": [{"class": "File", "location": "file:///example/d"}], + } + ), ] assert dedup(not_deduped) == expected diff --git a/tests/test_http_input.py b/tests/test_http_input.py index e80260ff9..d98e8a368 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -1,24 +1,27 @@ import os +from collections.abc import MutableSequence from datetime import datetime from pathlib import Path +from cwl_utils.types import CWLDirectoryType, CWLFileType from pytest_httpserver import HTTPServer from cwltool.pathmapper import PathMapper -from cwltool.utils import CWLObjectType def test_http_path_mapping(tmp_path: Path) -> None: input_file_path = ( "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta" ) - base_file: list[CWLObjectType] = [ - { - "class": "File", - "location": "https://raw.githubusercontent.com/common-workflow-language/" - "cwltool/main/tests/2.fasta", - "basename": "chr20.fa", - } + base_file: MutableSequence[CWLFileType | CWLDirectoryType] = [ + CWLFileType( + **{ + "class": "File", + "location": "https://raw.githubusercontent.com/common-workflow-language/" + "cwltool/main/tests/2.fasta", + "basename": "chr20.fa", + } + ) ] pathmap = PathMapper(base_file, os.getcwd(), str(tmp_path))._pathmap @@ -54,12 +57,14 @@ def test_modification_date(tmp_path: Path) -> None: ) location = httpserver.url_for(f"/{remote_file_name}") - base_file: list[CWLObjectType] = [ - { - "class": "File", - "location": location, - "basename": remote_file_name, - } + base_file: MutableSequence[CWLFileType | CWLDirectoryType] = [ + CWLFileType( + **{ + "class": "File", + "location": location, + "basename": remote_file_name, + } + ) ] date_now = datetime.now() diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index 2c5df6339..a7144a8f0 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -9,6 +9,7 @@ import pytest from cwl_utils import sandboxjs +from cwl_utils.types import CWLFileType from cwltool.factory import Factory from cwltool.loghandler import _logger, configure_logging @@ -41,7 +42,7 @@ def test_value_from_two_concatenated_expressions() -> None: js_engine.localdata = threading.local() # type: ignore[attr-defined] factory = Factory() echo = factory.make(get_data("tests/wf/vf-concat.cwl")) - file = {"class": "File", "location": get_data("tests/wf/whale.txt")} + file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")}) assert echo(file1=file) == {"out": "a string\n"} @@ -87,7 +88,7 @@ def test_value_from_two_concatenated_expressions_podman( with monkeypatch.context() as m: m.setenv("PATH", new_paths) echo = factory.make(get_data("tests/wf/vf-concat.cwl")) - file = {"class": "File", "location": get_data("tests/wf/whale.txt")} + file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")}) assert echo(file1=file) == {"out": "a string\n"} @@ -112,7 +113,7 @@ def test_value_from_two_concatenated_expressions_singularity( m.setenv("CWL_SINGULARITY_CACHE", str(singularity_cache)) m.setenv("PATH", new_paths) echo = factory.make(get_data("tests/wf/vf-concat.cwl")) - file = {"class": "File", "location": get_data("tests/wf/whale.txt")} + file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")}) assert echo(file1=file) == {"out": "a string\n"} diff --git a/tests/test_load_tool.py b/tests/test_load_tool.py index 2b9e9b6ed..c1c1ce017 100644 --- a/tests/test_load_tool.py +++ b/tests/test_load_tool.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest +from cwl_utils.types import CWLObjectType from schema_salad.exceptions import ValidationException from cwltool.context import LoadingContext, RuntimeContext @@ -13,7 +14,6 @@ from cwltool.loghandler import _logger, configure_logging from cwltool.process import use_custom_schema, use_standard_schema from cwltool.update import INTERNAL_VERSION -from cwltool.utils import CWLObjectType from .util import get_data diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 2d76e07a7..367714f27 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,6 +1,8 @@ import json from pathlib import Path +from cwl_utils.types import CWLFileType + from cwltool.context import RuntimeContext from cwltool.executors import MultithreadedJobExecutor from cwltool.factory import Factory @@ -17,7 +19,7 @@ def test_sequential_workflow(tmp_path: Path) -> None: runtime_context.select_resources = executor.select_resources factory = Factory(executor, None, runtime_context) echo = factory.make(get_data(test_file)) - file_contents = {"class": "File", "location": get_data("tests/wf/whale.txt")} + file_contents = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")}) assert echo(file1=file_contents) == {"count_output": 16} diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 096de9942..c01d84944 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -4,6 +4,7 @@ from typing import IO, Any, cast import pytest +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap @@ -14,7 +15,7 @@ from cwltool.main import main from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION -from cwltool.utils import CONTENT_LIMIT, CWLObjectType, bytes2str_in_dicts +from cwltool.utils import CONTENT_LIMIT, bytes2str_in_dicts from .util import needs_docker diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index 4ffac24bd..c0005cc77 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,14 +1,17 @@ +from collections.abc import MutableSequence + import pytest +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType from cwltool.pathmapper import PathMapper -from cwltool.utils import CWLObjectType, normalizeFilesDirs +from cwltool.utils import normalizeFilesDirs def test_subclass() -> None: class SubPathMapper(PathMapper): def __init__( self, - referenced_files: list[CWLObjectType], + referenced_files: MutableSequence[CWLFileType | CWLDirectoryType], basedir: str, stagedir: str, new: str, @@ -89,7 +92,7 @@ def test_basename_field_generation(filename: str, expected: tuple[str, str]) -> "nameext": nameext, } - my_file = {"class": "File", "location": "/foo/" + filename} + my_file = CWLFileType(**{"class": "File", "location": "/foo/" + filename}) normalizeFilesDirs(my_file) assert my_file == expected2 diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 206b001bd..b6c316ce1 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,13 +1,14 @@ import shutil import tempfile -from collections.abc import Callable +from collections.abc import Callable, MutableMapping from io import StringIO +from typing import cast import pytest +from cwl_utils.types import CWLObjectType, CWLOutputType from cwltool.main import main from cwltool.secrets import SecretStore -from cwltool.utils import CWLObjectType from .util import get_data, needs_docker, needs_singularity @@ -27,7 +28,7 @@ def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None: storage, obscured = secrets assert obscured["foo"] != "bar" assert obscured["baz"] == "quux" - result = storage.retrieve(obscured) + result = cast(MutableMapping[str, CWLOutputType], storage.retrieve(obscured)) assert isinstance(result, dict) and result["foo"] == "bar" diff --git a/tests/test_streaming.py b/tests/test_streaming.py index aef4a7efe..c68454409 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -5,6 +5,7 @@ from typing import cast import pytest +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap @@ -13,7 +14,6 @@ from cwltool.errors import WorkflowException from cwltool.job import JobBase from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION -from cwltool.utils import CWLObjectType from .util import get_data diff --git a/tox.ini b/tox.ini index 305603343..69d0d1437 100644 --- a/tox.ini +++ b/tox.ini @@ -4,9 +4,9 @@ envlist = py3{10,11,12,13,14}-unit py3{10,11,12,13,14}-bandit py3{10,11,12,13,14}-mypy - py312-lintreadme - py312-shellcheck - py312-pydocstyle + py313-lintreadme + py313-shellcheck + py313-pydocstyle skip_missing_interpreters = True @@ -31,9 +31,9 @@ description = py3{10,11,12,13,14}-lint: Lint the Python code py3{10,11,12,13,14}-bandit: Search for common security issues py3{10,11,12,13,14}-mypy: Check for type safety - py312-pydocstyle: docstring style checker - py312-shellcheck: syntax check for shell scripts - py312-lintreadme: Lint the README.rst→.md conversion + py313-pydocstyle: docstring style checker + py313-shellcheck: syntax check for shell scripts + py313-lintreadme: Lint the README.rst→.md conversion passenv = CI @@ -52,27 +52,27 @@ deps = py3{10,11,12,13,14}-lint: -rlint-requirements.txt py3{10,11,12,13,14}-bandit: bandit py3{10,11,12,13,14}-mypy: -rmypy-requirements.txt - py312-pydocstyle: pydocstyle - py312-pydocstyle: diff-cover - py312-lintreadme: twine - py312-lintreadme: build - py312-lintreadme: readme_renderer[rst] + py313-pydocstyle: pydocstyle + py313-pydocstyle: diff-cover + py313-lintreadme: twine + py313-lintreadme: build + py313-lintreadme: readme_renderer[rst] setenv = LC_ALL = C.UTF-8 HOME = {envtmpdir} commands_pre = - py312-lintreadme: python -m build --outdir {pkg_dir} + py313-lintreadme: python -m build --outdir {pkg_dir} commands = py3{10,11,12,13,14}-unit: make coverage-report coverage.xml PYTEST_EXTRA="{posargs}" py3{10,11,12,13,14}-bandit: bandit -r cwltool py3{10,11,12,13,14}-lint: make flake8 format-check codespell-check py3{10,11,12,13,14}-mypy: make mypy mypyc PYTEST_EXTRA="{posargs}" - py312-shellcheck: make shellcheck - py312-pydocstyle: make diff_pydocstyle_report - py312-lintreadme: twine check {pkg_dir}/* + py313-shellcheck: make shellcheck + py313-pydocstyle: make diff_pydocstyle_report + py313-lintreadme: twine check {pkg_dir}/* skip_install = py3{10,11,12,13,14}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true