From e824db710bf92d4ea499e47378b6d204859efa1a Mon Sep 17 00:00:00 2001 From: mjosse Date: Tue, 22 Oct 2024 15:10:47 +0200 Subject: [PATCH 01/17] try to fix branch mess --- lib/galaxy/model/store/ro_crate_utils.py | 210 +++++++++++++++++++++++ 1 file changed, 210 insertions(+) diff --git a/lib/galaxy/model/store/ro_crate_utils.py b/lib/galaxy/model/store/ro_crate_utils.py index f3592ac04ab9..08e0f5e69067 100644 --- a/lib/galaxy/model/store/ro_crate_utils.py +++ b/lib/galaxy/model/store/ro_crate_utils.py @@ -222,6 +222,216 @@ def _add_workflows(self, crate: ROCrate): crate.mainEntity["name"] = self.workflow.name crate.mainEntity["subjectOf"] = cwl_wf + # Adding multiple creators if available + if self.workflow.creator_metadata: + for creator_data in self.workflow.creator_metadata: + if creator_data.get("class") == "Person": + # Create the person entity + creator_entity = crate.add( + ContextEntity( + crate, + creator_data.get("identifier", ""), # Default to empty string if identifier is missing + properties={ + "@type": "Person", + "name": creator_data.get("name", ""), # Default to empty string if name is missing + "orcid": creator_data.get( + "identifier", "" + ), # Assuming identifier is ORCID, or adjust as needed + "url": creator_data.get("url", ""), # Add URL if available, otherwise empty string + "email": creator_data.get( + "email", "" + ), # Add email if available, otherwise empty string + }, + ) + ) + # Append the person creator entity to the mainEntity + crate.mainEntity.append_to("creator", creator_entity) + + elif creator_data.get("class") == "Organization": + # Create the organization entity + organization_entity = crate.add( + ContextEntity( + crate, + creator_data.get( + "url", "" + ), # Use URL as identifier if available, otherwise empty string + properties={ + "@type": "Organization", + "name": creator_data.get("name", ""), # Default to empty string if name is missing + "url": creator_data.get("url", ""), # Add URL if available, otherwise empty string + }, + ) + ) + # Append the organization entity to the mainEntity + crate.mainEntity.append_to("creator", organization_entity) + + # Add CWL workflow entity if exists + crate.mainEntity["subjectOf"] = cwl_wf + + # Add tools used in the workflow + self._add_tools(crate) + self._add_steps(crate) + + def _add_steps(self, crate: ROCrate): + """ + Add workflow steps (HowToStep) to the RO-Crate. These are unique for each tool occurrence. + """ + step_entities = [] + # Initialize the position as a list with a single element to keep it mutable + position = [1] + self._add_steps_recursive(self.workflow.steps, crate, step_entities, position) + return step_entities + + def _add_steps_recursive(self, steps, crate: ROCrate, step_entities, position): + """ + Recursively add HowToStep entities from workflow steps, ensuring that + the position index is maintained across subworkflows. + """ + for step in steps: + if step.type == "tool": + # Create a unique HowToStep entity for each step + step_id = f"step_{position[0]}" + step_description = None + if step.annotations: + annotations_list = [annotation.annotation for annotation in step.annotations if annotation] + step_description = " ".join(annotations_list) if annotations_list else None + + # Add HowToStep entity to the crate + step_entity = crate.add( + ContextEntity( + crate, + step_id, + properties={ + "@type": "HowToStep", + "position": position[0], + "name": step.tool_id, + "description": step_description, + }, + ) + ) + + # Append the HowToStep entity to the workflow steps list + step_entities.append(step_entity) + crate.mainEntity.append_to("step", step_entity) + + # Increment the position counter + position[0] += 1 + + # Handle subworkflows recursively + elif step.type == "subworkflow": + subworkflow = step.subworkflow + if subworkflow: + self._add_steps_recursive(subworkflow.steps, crate, step_entities, position) + + def _add_tools(self, crate: ROCrate): + tool_entities = [] + self._add_tools_recursive(self.workflow.steps, crate, tool_entities) + + def _add_tools_recursive(self, steps, crate: ROCrate, tool_entities): + """ + Recursively add SoftwareApplication entities from workflow steps, reusing tools when necessary. + """ + for step in steps: + if step.type == "tool": + tool_id = step.tool_id + tool_version = step.tool_version + + # Cache key based on tool ID and version + tool_key = f"{tool_id}:{tool_version}" + + # Check if tool entity is already in cache + if tool_key in self.tool_cache: + tool_entity = self.tool_cache[tool_key] + else: + # Create a new tool entity + tool_name = tool_id + tool_description = None + if step.annotations: + annotations_list = [annotation.annotation for annotation in step.annotations if annotation] + tool_description = " ".join(annotations_list) if annotations_list else None + + # Retrieve the tool metadata from the toolbox + tool_metadata = self._get_tool_metadata(tool_id) + + # Add tool entity to the RO-Crate + tool_entity = crate.add( + ContextEntity( + crate, + tool_id, + properties={ + "@type": "SoftwareApplication", + "name": tool_name, + "version": tool_version, + "description": tool_description, + "url": "https://toolshed.g2.bx.psu.edu", # URL if relevant + "citation": tool_metadata["citations"], + "identifier": tool_metadata["xrefs"], + "EDAM operation": tool_metadata["edam_operations"], + }, + ) + ) + + # Store the tool entity in the cache + self.tool_cache[tool_key] = tool_entity + + # Append the tool entity to the workflow (instrument) and store it in the list + tool_entities.append(tool_entity) + crate.mainEntity.append_to("instrument", tool_entity) + + # Handle subworkflows recursively + elif step.type == "subworkflow": + subworkflow = step.subworkflow + if subworkflow: + self._add_tools_recursive(subworkflow.steps, crate, tool_entities) + + def _get_tool_metadata(self, tool_id: str): + """ + Retrieve the tool metadata (citations, xrefs, EDAM operations) using the ToolBox. + + Args: + toolbox (ToolBox): An instance of the Galaxy ToolBox. + tool_id (str): The ID of the tool to retrieve metadata for. + + Returns: + dict: A dictionary containing citations, xrefs, and EDAM operations for the tool. + """ + tool = self.toolbox.get_tool(tool_id) + if not tool: + return None + + # Extracting relevant metadata from the tool object + citations = [] + if tool.citations: + for citation in tool.citations: + citations.append( + { + "type": citation.type, # e.g., "doi" or "bibtex" + "value": citation.value, # The actual DOI, BibTeX, etc. + } + ) + + xrefs = [] + if tool.xrefs: + for xref in tool.xrefs: + xrefs.append( + { + "type": xref.type, # e.g., "registry", "repository", etc. + "value": xref.value, # The identifier or link + } + ) + + # Handling EDAM operations, which are simple values in your XML + edam_operations = [] + if tool.edam_operations: + for operation in tool.edam_operations: + edam_operations.append({"value": operation}) # Extract the operation code (e.g., "operation_3482") + + return { + "citations": citations, # List of structured citation entries + "xrefs": xrefs, # List of structured xref entries + "edam_operations": edam_operations, # List of structured EDAM operations + } + def _add_create_action(self, crate: ROCrate): self.create_action = crate.add( ContextEntity( From a05e522b4d5e5383dd5b5313201ca7607a928a37 Mon Sep 17 00:00:00 2001 From: mjosse Date: Wed, 23 Oct 2024 11:30:33 +0200 Subject: [PATCH 02/17] add cache --- lib/galaxy/model/store/ro_crate_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/model/store/ro_crate_utils.py b/lib/galaxy/model/store/ro_crate_utils.py index 08e0f5e69067..deff1288848e 100644 --- a/lib/galaxy/model/store/ro_crate_utils.py +++ b/lib/galaxy/model/store/ro_crate_utils.py @@ -45,6 +45,7 @@ class WorkflowRunCrateProfileBuilder: def __init__(self, model_store: Any): self.model_store = model_store + self.toolbox = self.model_store.app.toolbox self.invocation: WorkflowInvocation = model_store.included_invocations[0] self.workflow: Workflow = self.invocation.workflow self.param_type_mapping = { @@ -85,6 +86,8 @@ def __init__(self, model_store: Any): self.file_entities: Dict[int, Any] = {} self.param_entities: Dict[int, Any] = {} self.pv_entities: Dict[str, Any] = {} + # Cache for tools to avoid duplicating entities for the same tool + self.tool_cache: Dict[str, ContextEntity] = {} def build_crate(self): crate = ROCrate() From effa82eabe9974a220f09df437de06885fc8fe64 Mon Sep 17 00:00:00 2001 From: mjosse Date: Thu, 31 Oct 2024 11:34:31 +0100 Subject: [PATCH 03/17] add some tests --- test/unit/data/model/test_model_store.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index 99dab6ddcb9e..6415cd690e26 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -19,6 +19,7 @@ from rocrate.rocrate import ROCrate from sqlalchemy import select from sqlalchemy.orm.scoping import scoped_session +from unittest.mock import Mock from galaxy import model from galaxy.model import store @@ -1199,11 +1200,33 @@ def write_composite_file(self, dataset_instance, contents, file_name): ) +class MockTool: + """Mock class to simulate Galaxy tools with essential metadata for testing.""" + + def __init__(self, tool_id): + self.tool_id = tool_id + self.citations = [{"type": "doi", "value": "10.1234/example.doi"}] + self.xrefs = [{"type": "registry", "value": "tool_registry_id"}] + self.edam_operations = ["operation_1234"] + + +class MockToolbox: + """Mock class for the Galaxy toolbox, which returns tools based on their IDs.""" + + def get_tool(self, tool_id): + # Returns a MockTool object with basic metadata for testing + return MockTool(tool_id) + + def _mock_app(store_by=DEFAULT_OBJECT_STORE_BY): app = TestApp() test_object_store_config = TestConfig(store_by=store_by) app.object_store = test_object_store_config.object_store app.model.Dataset.object_store = app.object_store + + # Add a mocked toolbox attribute for tests requiring tool metadata + app.toolbox = MockToolbox() + return app From 468de8a37e24e6b19fa90ee4a26509366f5d451b Mon Sep 17 00:00:00 2001 From: mjosse Date: Thu, 31 Oct 2024 11:43:29 +0100 Subject: [PATCH 04/17] fix lint rm useless import --- test/unit/data/model/test_model_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index 6415cd690e26..1c0279e18eb9 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -19,7 +19,6 @@ from rocrate.rocrate import ROCrate from sqlalchemy import select from sqlalchemy.orm.scoping import scoped_session -from unittest.mock import Mock from galaxy import model from galaxy.model import store From 803a5586206de52366c561e2741a654a28efb11c Mon Sep 17 00:00:00 2001 From: mjosse Date: Sun, 24 Nov 2024 17:18:21 +0100 Subject: [PATCH 05/17] rollback to changes without tool metadata --- lib/galaxy/model/store/ro_crate_utils.py | 55 ------------------------ 1 file changed, 55 deletions(-) diff --git a/lib/galaxy/model/store/ro_crate_utils.py b/lib/galaxy/model/store/ro_crate_utils.py index deff1288848e..c2f8ba1e0ba5 100644 --- a/lib/galaxy/model/store/ro_crate_utils.py +++ b/lib/galaxy/model/store/ro_crate_utils.py @@ -45,7 +45,6 @@ class WorkflowRunCrateProfileBuilder: def __init__(self, model_store: Any): self.model_store = model_store - self.toolbox = self.model_store.app.toolbox self.invocation: WorkflowInvocation = model_store.included_invocations[0] self.workflow: Workflow = self.invocation.workflow self.param_type_mapping = { @@ -353,9 +352,6 @@ def _add_tools_recursive(self, steps, crate: ROCrate, tool_entities): annotations_list = [annotation.annotation for annotation in step.annotations if annotation] tool_description = " ".join(annotations_list) if annotations_list else None - # Retrieve the tool metadata from the toolbox - tool_metadata = self._get_tool_metadata(tool_id) - # Add tool entity to the RO-Crate tool_entity = crate.add( ContextEntity( @@ -367,9 +363,6 @@ def _add_tools_recursive(self, steps, crate: ROCrate, tool_entities): "version": tool_version, "description": tool_description, "url": "https://toolshed.g2.bx.psu.edu", # URL if relevant - "citation": tool_metadata["citations"], - "identifier": tool_metadata["xrefs"], - "EDAM operation": tool_metadata["edam_operations"], }, ) ) @@ -387,54 +380,6 @@ def _add_tools_recursive(self, steps, crate: ROCrate, tool_entities): if subworkflow: self._add_tools_recursive(subworkflow.steps, crate, tool_entities) - def _get_tool_metadata(self, tool_id: str): - """ - Retrieve the tool metadata (citations, xrefs, EDAM operations) using the ToolBox. - - Args: - toolbox (ToolBox): An instance of the Galaxy ToolBox. - tool_id (str): The ID of the tool to retrieve metadata for. - - Returns: - dict: A dictionary containing citations, xrefs, and EDAM operations for the tool. - """ - tool = self.toolbox.get_tool(tool_id) - if not tool: - return None - - # Extracting relevant metadata from the tool object - citations = [] - if tool.citations: - for citation in tool.citations: - citations.append( - { - "type": citation.type, # e.g., "doi" or "bibtex" - "value": citation.value, # The actual DOI, BibTeX, etc. - } - ) - - xrefs = [] - if tool.xrefs: - for xref in tool.xrefs: - xrefs.append( - { - "type": xref.type, # e.g., "registry", "repository", etc. - "value": xref.value, # The identifier or link - } - ) - - # Handling EDAM operations, which are simple values in your XML - edam_operations = [] - if tool.edam_operations: - for operation in tool.edam_operations: - edam_operations.append({"value": operation}) # Extract the operation code (e.g., "operation_3482") - - return { - "citations": citations, # List of structured citation entries - "xrefs": xrefs, # List of structured xref entries - "edam_operations": edam_operations, # List of structured EDAM operations - } - def _add_create_action(self, crate: ROCrate): self.create_action = crate.add( ContextEntity( From f26662a9c961d922b55884549cb434e749febe27 Mon Sep 17 00:00:00 2001 From: mjosse Date: Sun, 24 Nov 2024 17:25:53 +0100 Subject: [PATCH 06/17] remove useless tests --- test/unit/data/model/test_model_store.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index 1c0279e18eb9..d86bf0e633bb 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -1199,33 +1199,12 @@ def write_composite_file(self, dataset_instance, contents, file_name): ) -class MockTool: - """Mock class to simulate Galaxy tools with essential metadata for testing.""" - - def __init__(self, tool_id): - self.tool_id = tool_id - self.citations = [{"type": "doi", "value": "10.1234/example.doi"}] - self.xrefs = [{"type": "registry", "value": "tool_registry_id"}] - self.edam_operations = ["operation_1234"] - - -class MockToolbox: - """Mock class for the Galaxy toolbox, which returns tools based on their IDs.""" - - def get_tool(self, tool_id): - # Returns a MockTool object with basic metadata for testing - return MockTool(tool_id) - - def _mock_app(store_by=DEFAULT_OBJECT_STORE_BY): app = TestApp() test_object_store_config = TestConfig(store_by=store_by) app.object_store = test_object_store_config.object_store app.model.Dataset.object_store = app.object_store - # Add a mocked toolbox attribute for tests requiring tool metadata - app.toolbox = MockToolbox() - return app From a582e5281c497e96dde312f574993952ebe5d9e9 Mon Sep 17 00:00:00 2001 From: mjosse Date: Mon, 25 Nov 2024 15:18:54 +0100 Subject: [PATCH 07/17] add some tests --- test/unit/data/model/test_model_store.py | 164 ++++++++++++++++++----- 1 file changed, 134 insertions(+), 30 deletions(-) diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index d86bf0e633bb..56c77bece0d7 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -442,7 +442,7 @@ def test_import_export_library(): def test_import_export_invocation(): app = _mock_app() workflow_invocation = _setup_invocation(app) - + print(workflow_invocation) temp_directory = mkdtemp() with store.DirectoryModelExportStore(temp_directory, app=app) as export_store: export_store.export_workflow_invocation(workflow_invocation) @@ -484,6 +484,56 @@ def validate_has_mit_license(ro_crate: ROCrate): assert found_license +def validate_creators(ro_crate: ROCrate): + """ + Validate that creators (Person and Organization) are correctly added. + """ + creators = ro_crate.mainEntity.get("creator") + assert creators, "No creators found in the RO-Crate" + + for creator in creators: + assert creator["@type"] in {"Person", "Organization"} + if creator["@type"] == "Person": + assert "name" in creator + assert "orcid" in creator or "identifier" in creator + assert "email" in creator + elif creator["@type"] == "Organization": + assert "name" in creator + assert "url" in creator + + +def validate_steps(ro_crate: ROCrate): + """ + Validate that workflow steps (HowToStep) are correctly added. + """ + steps = ro_crate.mainEntity.get("step") + assert steps, "No steps found in the RO-Crate" + + for i, step in enumerate(steps, start=1): + assert step["@type"] == "HowToStep" + assert step["position"] == i + assert "name" in step + assert "description" in step or step["description"] is None + + +def validate_tools(ro_crate: ROCrate): + """ + Validate that tools (SoftwareApplication) are correctly added. + """ + tools = ro_crate.mainEntity.get("instrument") + assert tools, "No tools found in the RO-Crate" + + tool_ids = set() + for tool in tools: + assert tool["@type"] == "SoftwareApplication" + assert "name" in tool + assert "version" in tool + assert "url" in tool + assert "description" in tool or tool["description"] is None + assert tool.id not in tool_ids, "Duplicate tool found" + tool_ids.add(tool.id) + + def validate_has_readme(ro_crate: ROCrate): found_readme = False for e in ro_crate.get_entities(): @@ -566,6 +616,9 @@ def validate_invocation_crate_directory(crate_directory): validate_has_pl_galaxy(crate) validate_organize_action(crate) validate_has_mit_license(crate) + validate_creators(crate) + validate_steps(crate) + validate_tools(crate) # validate_has_readme(crate) @@ -983,31 +1036,64 @@ def _setup_simple_cat_job(app, state="ok"): def _setup_invocation(app): sa_session = app.model.context + # Set up a user, history, datasets, and job u, h, d1, d2, j = _setup_simple_cat_job(app) j.parameters = [model.JobParameter(name="index_path", value='"/old/path/human"')] + # Create a workflow + workflow = model.Workflow() + workflow.license = "MIT" + workflow.name = "Test Workflow" + workflow.creator_metadata = [ + {"class": "Person", "name": "Alice", "identifier": "0000-0001-2345-6789", "email": "alice@example.com"}, + ] + + # Create and associate a data_input step workflow_step_1 = model.WorkflowStep() workflow_step_1.order_index = 0 workflow_step_1.type = "data_input" - sa_session.add(workflow_step_1) - workflow_1 = _workflow_from_steps(u, [workflow_step_1]) - workflow_1.license = "MIT" - workflow_1.name = "Test Workflow" - sa_session.add(workflow_1) - workflow_invocation = _invocation_for_workflow(u, workflow_1) - invocation_step = model.WorkflowInvocationStep() - invocation_step.workflow_step = workflow_step_1 - invocation_step.job = j - sa_session.add(invocation_step) - output_assoc = model.WorkflowInvocationStepOutputDatasetAssociation() - output_assoc.dataset = d2 - invocation_step.output_datasets = [output_assoc] - workflow_invocation.steps = [invocation_step] + workflow_step_1.label = "Input Step" + workflow.steps.append(workflow_step_1) + sa_session.add(workflow_step_1) # Persist step in the session + + # Create and associate a tool step + workflow_step_2 = model.WorkflowStep() + workflow_step_2.order_index = 0 + workflow_step_2.type = "tool" + workflow_step_2.tool_id = "example_tool" + workflow_step_2.tool_version = "1.0" + workflow_step_2.label = "Example Tool Step" + workflow.steps.append(workflow_step_2) + sa_session.add(workflow_step_2) # Persist step in the session + + sa_session.add(workflow) # Persist the workflow itself + + # Create a workflow invocation + workflow_invocation = _invocation_for_workflow(u, workflow) + + # Associate invocation step for data_input + invocation_step_1 = model.WorkflowInvocationStep() + invocation_step_1.workflow_step = workflow_step_1 + invocation_step_1.job = j + sa_session.add(invocation_step_1) + + # Associate invocation step for tool + invocation_step_2 = model.WorkflowInvocationStep() + invocation_step_2.workflow_step = workflow_step_2 + sa_session.add(invocation_step_2) + + # Add steps to the invocation + workflow_invocation.steps = [invocation_step_1, invocation_step_2] workflow_invocation.user = u workflow_invocation.add_input(d1, step=workflow_step_1) - wf_output = model.WorkflowOutput(workflow_step_1, label="output_label") - workflow_invocation.add_output(wf_output, workflow_step_1, d2) + + # Add workflow output associated with the tool step + wf_output = model.WorkflowOutput(workflow_step_2, label="output_label") + workflow_invocation.add_output(wf_output, workflow_step_2, d2) + + # Commit the workflow and invocation app.add_and_commit(workflow_invocation) + return workflow_invocation @@ -1086,27 +1172,45 @@ def _setup_collection_invocation(app): def _setup_simple_invocation(app): sa_session = app.model.context + # Set up a simple user, history, datasets, and job u, h, d1, d2, j = _setup_simple_cat_job(app) j.parameters = [model.JobParameter(name="index_path", value='"/old/path/human"')] - workflow_step_1 = model.WorkflowStep() - workflow_step_1.order_index = 0 - workflow_step_1.type = "data_input" - workflow_step_1.tool_inputs = {} # type:ignore[assignment] - sa_session.add(workflow_step_1) - workflow = _workflow_from_steps(u, [workflow_step_1]) + # Create a workflow + workflow = model.Workflow() workflow.license = "MIT" workflow.name = "Test Workflow" - workflow.create_time = now() - workflow.update_time = now() + workflow.creator_metadata = [ + {"class": "Person", "name": "Bob", "identifier": "0000-0002-3456-7890", "email": "bob@example.com"}, + ] + + # Create and associate a data_input step + workflow_step_input = model.WorkflowStep() + workflow_step_input.order_index = 0 + workflow_step_input.type = "data_input" + workflow_step_input.label = "Input Step" + workflow.steps.append(workflow_step_input) + + # Create and associate a tool step + workflow_step_tool = model.WorkflowStep() + workflow_step_tool.order_index = 1 + workflow_step_tool.type = "tool" + workflow_step_tool.tool_id = "example_tool" + workflow_step_tool.tool_version = "1.0" + workflow_step_tool.label = "Example Tool Step" + workflow.steps.append(workflow_step_tool) + sa_session.add(workflow) + + # Create a workflow invocation invocation = _invocation_for_workflow(u, workflow) - invocation.create_time = now() - invocation.update_time = now() + invocation.add_input(d1, step=workflow_step_input) # Associate input dataset + wf_output = model.WorkflowOutput(workflow_step_tool, label="output_label") + invocation.add_output(wf_output, workflow_step_tool, d2) # Associate output dataset + + # Commit the workflow and invocation to the database + app.add_and_commit(invocation) - invocation.add_input(d1, step=workflow_step_1) - wf_output = model.WorkflowOutput(workflow_step_1, label="output_label") - invocation.add_output(wf_output, workflow_step_1, d2) return invocation From 5a26dc2d9d9864e5769dde497502e995416e9b65 Mon Sep 17 00:00:00 2001 From: mjosse Date: Mon, 25 Nov 2024 16:24:17 +0100 Subject: [PATCH 08/17] try to fix conflicts --- test/unit/data/model/test_model_store.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index 56c77bece0d7..1a69fef03130 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -1177,20 +1177,19 @@ def _setup_simple_invocation(app): j.parameters = [model.JobParameter(name="index_path", value='"/old/path/human"')] # Create a workflow - workflow = model.Workflow() + workflow_step_1 = model.WorkflowStep() + workflow_step_1.order_index = 0 + workflow_step_1.type = "data_input" + workflow_step_1.label = "Input Step" + workflow_step_1.tool_inputs = {} + sa_session.add(workflow_step_1) + workflow = _workflow_from_steps(u, [workflow_step_1]) workflow.license = "MIT" workflow.name = "Test Workflow" workflow.creator_metadata = [ {"class": "Person", "name": "Bob", "identifier": "0000-0002-3456-7890", "email": "bob@example.com"}, ] - # Create and associate a data_input step - workflow_step_input = model.WorkflowStep() - workflow_step_input.order_index = 0 - workflow_step_input.type = "data_input" - workflow_step_input.label = "Input Step" - workflow.steps.append(workflow_step_input) - # Create and associate a tool step workflow_step_tool = model.WorkflowStep() workflow_step_tool.order_index = 1 @@ -1204,7 +1203,7 @@ def _setup_simple_invocation(app): # Create a workflow invocation invocation = _invocation_for_workflow(u, workflow) - invocation.add_input(d1, step=workflow_step_input) # Associate input dataset + invocation.add_input(d1, step=workflow_step_1) # Associate input dataset wf_output = model.WorkflowOutput(workflow_step_tool, label="output_label") invocation.add_output(wf_output, workflow_step_tool, d2) # Associate output dataset From 875e8ea033419c64e5bf77883daae3f06e03982f Mon Sep 17 00:00:00 2001 From: mjosse Date: Mon, 25 Nov 2024 16:28:44 +0100 Subject: [PATCH 09/17] rm label --- test/unit/data/model/test_model_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index 1a69fef03130..1b75b6937b41 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -1180,7 +1180,6 @@ def _setup_simple_invocation(app): workflow_step_1 = model.WorkflowStep() workflow_step_1.order_index = 0 workflow_step_1.type = "data_input" - workflow_step_1.label = "Input Step" workflow_step_1.tool_inputs = {} sa_session.add(workflow_step_1) workflow = _workflow_from_steps(u, [workflow_step_1]) From 58219b1734dc1049a874a69e4df24638f4d3d5f5 Mon Sep 17 00:00:00 2001 From: mjosse Date: Tue, 26 Nov 2024 09:15:09 +0100 Subject: [PATCH 10/17] rm useless import --- test/unit/data/model/test_model_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index 1b75b6937b41..abae876244ef 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -24,7 +24,6 @@ from galaxy.model import store from galaxy.model.base import transaction from galaxy.model.metadata import MetadataTempFile -from galaxy.model.orm.now import now from galaxy.model.unittest_utils import GalaxyDataTestApp from galaxy.model.unittest_utils.store_fixtures import ( deferred_hda_model_store_dict, From f9303588168994d180075bcb0c80961b81b91806 Mon Sep 17 00:00:00 2001 From: mjosse Date: Wed, 27 Nov 2024 16:01:54 +0100 Subject: [PATCH 11/17] fix mypy errors --- lib/galaxy/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 64e3c143e736..5e18bbfb3c50 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7815,7 +7815,7 @@ class Workflow(Base, Dictifiable, RepresentById): has_cycles: Mapped[Optional[bool]] has_errors: Mapped[Optional[bool]] reports_config: Mapped[Optional[bytes]] = mapped_column(JSONType) - creator_metadata: Mapped[Optional[bytes]] = mapped_column(JSONType) + creator_metadata: Mapped[Optional[List[Dict[str, Any]]]] = mapped_column(JSONType) license: Mapped[Optional[str]] = mapped_column(TEXT) source_metadata: Mapped[Optional[bytes]] = mapped_column(JSONType) uuid: Mapped[Optional[Union[UUID, str]]] = mapped_column(UUIDType) From 124db0b87053cb1d458786cd5cf8c8e083fbb6bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marie=20Joss=C3=A9?= <84919248+Marie59@users.noreply.github.com> Date: Wed, 27 Nov 2024 16:31:31 +0100 Subject: [PATCH 12/17] Update lib/galaxy/model/store/ro_crate_utils.py Co-authored-by: Nicola Soranzo --- lib/galaxy/model/store/ro_crate_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/store/ro_crate_utils.py b/lib/galaxy/model/store/ro_crate_utils.py index c2f8ba1e0ba5..0a2dcd991907 100644 --- a/lib/galaxy/model/store/ro_crate_utils.py +++ b/lib/galaxy/model/store/ro_crate_utils.py @@ -278,7 +278,7 @@ def _add_steps(self, crate: ROCrate): """ Add workflow steps (HowToStep) to the RO-Crate. These are unique for each tool occurrence. """ - step_entities = [] + step_entities: List[ContextEntity] = [] # Initialize the position as a list with a single element to keep it mutable position = [1] self._add_steps_recursive(self.workflow.steps, crate, step_entities, position) From 0cbc5084134755be46d22b374ac946da69cb5381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marie=20Joss=C3=A9?= <84919248+Marie59@users.noreply.github.com> Date: Wed, 27 Nov 2024 16:31:45 +0100 Subject: [PATCH 13/17] Update lib/galaxy/model/store/ro_crate_utils.py Co-authored-by: Nicola Soranzo --- lib/galaxy/model/store/ro_crate_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/store/ro_crate_utils.py b/lib/galaxy/model/store/ro_crate_utils.py index 0a2dcd991907..842911513996 100644 --- a/lib/galaxy/model/store/ro_crate_utils.py +++ b/lib/galaxy/model/store/ro_crate_utils.py @@ -326,7 +326,7 @@ def _add_steps_recursive(self, steps, crate: ROCrate, step_entities, position): self._add_steps_recursive(subworkflow.steps, crate, step_entities, position) def _add_tools(self, crate: ROCrate): - tool_entities = [] + tool_entities: List[ContextEntity] = [] self._add_tools_recursive(self.workflow.steps, crate, tool_entities) def _add_tools_recursive(self, steps, crate: ROCrate, tool_entities): From f585677476a8f2093b5a779b554bb4c217b46d42 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 27 Nov 2024 16:27:52 +0000 Subject: [PATCH 14/17] Add missing import --- lib/galaxy/model/store/ro_crate_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/model/store/ro_crate_utils.py b/lib/galaxy/model/store/ro_crate_utils.py index 842911513996..90fb617d3ec1 100644 --- a/lib/galaxy/model/store/ro_crate_utils.py +++ b/lib/galaxy/model/store/ro_crate_utils.py @@ -3,6 +3,7 @@ from typing import ( Any, Dict, + List, Optional, ) From e66652e78423158932bd6574568bbeea21e2fb8d Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Thu, 28 Nov 2024 14:47:36 +0100 Subject: [PATCH 15/17] Drop stray print statement --- test/unit/data/model/test_model_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index abae876244ef..fd874732da24 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -441,7 +441,6 @@ def test_import_export_library(): def test_import_export_invocation(): app = _mock_app() workflow_invocation = _setup_invocation(app) - print(workflow_invocation) temp_directory = mkdtemp() with store.DirectoryModelExportStore(temp_directory, app=app) as export_store: export_store.export_workflow_invocation(workflow_invocation) From f1b404ddc03854f88f151823a4abd06e856dcf64 Mon Sep 17 00:00:00 2001 From: mjosse Date: Mon, 23 Dec 2024 09:51:32 +0100 Subject: [PATCH 16/17] imrpove ro-crate with eli's review --- lib/galaxy/model/store/ro_crate_utils.py | 6 +++--- test/unit/data/model/test_model_store.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/model/store/ro_crate_utils.py b/lib/galaxy/model/store/ro_crate_utils.py index 90fb617d3ec1..c53601fd8357 100644 --- a/lib/galaxy/model/store/ro_crate_utils.py +++ b/lib/galaxy/model/store/ro_crate_utils.py @@ -309,6 +309,7 @@ def _add_steps_recursive(self, steps, crate: ROCrate, step_entities, position): "position": position[0], "name": step.tool_id, "description": step_description, + "workExample": f"#{step.tool_id}" }, ) ) @@ -357,13 +358,12 @@ def _add_tools_recursive(self, steps, crate: ROCrate, tool_entities): tool_entity = crate.add( ContextEntity( crate, - tool_id, + f"#{tool_id}", # Prepend # to tool_id properties={ "@type": "SoftwareApplication", "name": tool_name, "version": tool_version, "description": tool_description, - "url": "https://toolshed.g2.bx.psu.edu", # URL if relevant }, ) ) @@ -373,7 +373,7 @@ def _add_tools_recursive(self, steps, crate: ROCrate, tool_entities): # Append the tool entity to the workflow (instrument) and store it in the list tool_entities.append(tool_entity) - crate.mainEntity.append_to("instrument", tool_entity) + crate.mainEntity.append_to("hasPart", tool_entity) # Handle subworkflows recursively elif step.type == "subworkflow": diff --git a/test/unit/data/model/test_model_store.py b/test/unit/data/model/test_model_store.py index fd874732da24..52a0ffa4e049 100644 --- a/test/unit/data/model/test_model_store.py +++ b/test/unit/data/model/test_model_store.py @@ -518,7 +518,7 @@ def validate_tools(ro_crate: ROCrate): """ Validate that tools (SoftwareApplication) are correctly added. """ - tools = ro_crate.mainEntity.get("instrument") + tools = ro_crate.mainEntity.get("hasPart") assert tools, "No tools found in the RO-Crate" tool_ids = set() @@ -526,7 +526,6 @@ def validate_tools(ro_crate: ROCrate): assert tool["@type"] == "SoftwareApplication" assert "name" in tool assert "version" in tool - assert "url" in tool assert "description" in tool or tool["description"] is None assert tool.id not in tool_ids, "Duplicate tool found" tool_ids.add(tool.id) From 9a2cea85776bdfdaf99365ebf8639885dbde1ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marie=20Joss=C3=A9?= <84919248+Marie59@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:01:13 +0100 Subject: [PATCH 17/17] Update lib/galaxy/model/store/ro_crate_utils.py Co-authored-by: Eli Chadwick --- lib/galaxy/model/store/ro_crate_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/store/ro_crate_utils.py b/lib/galaxy/model/store/ro_crate_utils.py index c53601fd8357..3f0b7cc57ad7 100644 --- a/lib/galaxy/model/store/ro_crate_utils.py +++ b/lib/galaxy/model/store/ro_crate_utils.py @@ -237,7 +237,7 @@ def _add_workflows(self, crate: ROCrate): properties={ "@type": "Person", "name": creator_data.get("name", ""), # Default to empty string if name is missing - "orcid": creator_data.get( + "identifier": creator_data.get( "identifier", "" ), # Assuming identifier is ORCID, or adjust as needed "url": creator_data.get("url", ""), # Add URL if available, otherwise empty string