From 6c2fbb3c79a53d4f7275028febb5ca17fc1c318d Mon Sep 17 00:00:00 2001 From: Kalev Takkis Date: Wed, 18 Sep 2024 15:12:11 +0100 Subject: [PATCH] feat: added source files to database and API Stored in Experiment model and avaliable in api/experiments/ --- api/urls.py | 1 + viewer/filters.py | 7 ++++ viewer/models.py | 4 +++ viewer/serializers.py | 6 ++++ viewer/target_loader.py | 75 ++++++++++++++++++++++++++++------------- viewer/views.py | 7 ++++ 6 files changed, 77 insertions(+), 23 deletions(-) diff --git a/api/urls.py b/api/urls.py index 50ee274e..c88fb765 100644 --- a/api/urls.py +++ b/api/urls.py @@ -113,6 +113,7 @@ "site_observations", viewer_views.SiteObservationView, basename='site_observations' ) router.register("canon_sites", viewer_views.CanonSiteView, basename='canon_sites') +router.register("experiments", viewer_views.ExperimentView, basename='experiments') router.register( "canon_site_confs", viewer_views.CanonSiteConfView, basename='canon_site_confs' ) diff --git a/viewer/filters.py b/viewer/filters.py index d0df524a..4c24ba70 100644 --- a/viewer/filters.py +++ b/viewer/filters.py @@ -7,6 +7,7 @@ CanonSite, CanonSiteConf, Compound, + Experiment, Pose, QuatAssembly, SiteObservation, @@ -60,6 +61,12 @@ class Meta: fields = ("target",) +class ExperimentFilter(TargetFilterMixin): + class Meta: + model = Experiment + fields = ("target",) + + class CanonSiteConfFilter(TargetFilterMixin): class Meta: model = CanonSiteConf diff --git a/viewer/models.py b/viewer/models.py index 3cb5e5f3..e104399a 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -215,13 +215,17 @@ class Experiment(models.Model): pdb_info = models.FileField( upload_to="target_loader_data/", null=True, max_length=255 ) + pdb_info_source_file = models.TextField(null=True) mtz_info = models.FileField( upload_to="target_loader_data/", null=True, max_length=255 ) + mtz_info_source_file = models.TextField(null=True) cif_info = models.FileField( upload_to="target_loader_data/", null=True, max_length=255 ) + cif_info_source_file = models.TextField(null=True) map_info = ArrayField(models.FileField(max_length=255), null=True) + map_info_source_files = ArrayField(models.TextField(null=True), null=True) type = models.PositiveSmallIntegerField(null=True) pdb_sha256 = models.TextField(null=True) prefix_tooltip = models.TextField(null=True) diff --git a/viewer/serializers.py b/viewer/serializers.py index 505280ca..5d070565 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -931,6 +931,12 @@ class Meta: fields = '__all__' +class ExperimentReadSerializer(serializers.ModelSerializer): + class Meta: + model = models.Experiment + fields = '__all__' + + class CanonSiteConfReadSerializer(serializers.ModelSerializer): class Meta: model = models.CanonSiteConf diff --git a/viewer/target_loader.py b/viewer/target_loader.py index 0c434906..fb61054c 100644 --- a/viewer/target_loader.py +++ b/viewer/target_loader.py @@ -523,7 +523,7 @@ def validate_map_files( obj_identifier: str, file_struct: list, validate_files: bool = True, - ) -> list[str]: + ) -> tuple[list[str], list[str]]: """Validate list of panddas event files. Special case of file validation, too complex to squeeze into @@ -533,17 +533,20 @@ def validate_map_files( def logfunc(_, message): self.report.log(logging.WARNING, message) - result = [] + paths = [] + source_files = [] for item in file_struct: fname, file_hash = self._check_file(item, obj_identifier, key, logfunc) + source_file = item.get("source_file", None) if not fname: continue if validate_files: self._check_file_hash(obj_identifier, key, fname, file_hash, logfunc) - result.append(fname) + paths.append(fname) + source_files.append(source_file) - return result + return paths, source_files def validate_files( self, @@ -552,7 +555,7 @@ def validate_files( required: Iterable[str] = (), recommended: Iterable[str] = (), validate_files: bool = True, - ) -> list[str | None]: + ) -> list[tuple[str | None, str | None]]: """Check if file exists and if sha256 hash matches (if given). file struct can come in 2 configurations: @@ -594,7 +597,7 @@ def logfunc(key, message): # schema isn't looking for this file, ignore continue - filename, file_hash = None, None + filename, file_hash, source_file = None, None, None # sort out the filename if isinstance(value, dict): @@ -604,6 +607,8 @@ def logfunc(key, message): if not filename: continue + source_file = value.get("source_file", None) + if validate_files: self._check_file_hash( obj_identifier, key, filename, file_hash, logfunc @@ -627,7 +632,7 @@ def logfunc(key, message): files = [] for f in list(required) + list(recommended): try: - files.append(result[f]) + files.append((result[f], source_file)) except KeyError: logfunc( f, @@ -637,7 +642,7 @@ def logfunc(key, message): METADATA_FILE, ), ) - files.append(None) # type: ignore [arg-type] + files.append((None, None)) # type: ignore [arg-type] logger.debug("Returning files: %s", files) @@ -753,9 +758,9 @@ def process_experiment( ) ( # pylint: disable=unbalanced-tuple-unpacking - pdb_info, - mtz_info, - cif_info, + pdb_info_t, + mtz_info_t, + cif_info_t, ) = self.validate_files( obj_identifier=experiment_name, file_struct=data["crystallographic_files"], @@ -767,12 +772,16 @@ def process_experiment( validate_files=validate_files, ) + pdb_info, pdb_info_source_file = pdb_info_t + mtz_info, mtz_info_source_file = mtz_info_t + cif_info, cif_info_source_file = cif_info_t + try: event_files = data["crystallographic_files"]["ligand_binding_events"] except KeyError: event_files = [] - map_info_files = self.validate_map_files( + map_info_files, map_info_source_files = self.validate_map_files( key="ligand_binding_events", obj_identifier=experiment_name, file_struct=event_files, @@ -836,7 +845,11 @@ def process_experiment( "pdb_info": str(self._get_final_path(pdb_info)), "mtz_info": str(self._get_final_path(mtz_info)), "cif_info": str(self._get_final_path(cif_info)), + "pdb_info_source_file": pdb_info_source_file, + "mtz_info_source_file": mtz_info_source_file, + "cif_info_source_file": cif_info_source_file, "map_info": map_info_paths, + "map_info_source_files": map_info_source_files, "prefix_tooltip": prefix_tooltip, "code_prefix": code_prefix, # this doesn't seem to be present @@ -1351,17 +1364,17 @@ def process_site_observation( xtalform_site = xtalform_sites[v_key] ( # pylint: disable=unbalanced-tuple-unpacking - bound_file, - apo_solv_file, - apo_desolv_file, - apo_file, - artefacts_file, - sigmaa_file, - diff_file, - event_file, - ligand_pdb, - ligand_mol, - ligand_smiles, + bound_file_t, + apo_solv_file_t, + apo_desolv_file_t, + apo_file_t, + artefacts_file_t, + sigmaa_file_t, + diff_file_t, + event_file_t, + ligand_pdb_t, + ligand_mol_t, + ligand_smiles_t, ) = self.validate_files( obj_identifier=experiment_id, file_struct=data, @@ -1383,6 +1396,18 @@ def process_site_observation( validate_files=validate_files, ) + bound_file = bound_file_t[0] + apo_solv_file = apo_solv_file_t[0] + apo_desolv_file = apo_desolv_file_t[0] + apo_file = apo_file_t[0] + artefacts_file = artefacts_file_t[0] + sigmaa_file = sigmaa_file_t[0] + diff_file = diff_file_t[0] + event_file = event_file_t[0] + ligand_pdb = ligand_pdb_t[0] + ligand_mol = ligand_mol_t[0] + ligand_smiles = ligand_smiles_t[0] + fields = { # Code for this protein (e.g. Mpro_Nterm-x0029_A_501_0) "longcode": longcode, @@ -1548,6 +1573,10 @@ def process_bundle(self): required=(TRANS_NEIGHBOURHOOD, TRANS_CONF_SITE, TRANS_REF_STRUCT), ) + trans_neighbourhood = trans_neighbourhood[0] + trans_conf_site = trans_conf_site[0] + trans_ref_struct = trans_ref_struct[0] + self.experiment_upload.project = self.project self.experiment_upload.target = self.target self.experiment_upload.committer = committer diff --git a/viewer/views.py b/viewer/views.py index a4a90748..b59cfc49 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -1741,6 +1741,13 @@ class CanonSiteView(ISPyBSafeQuerySet): ) +class ExperimentView(ISPyBSafeQuerySet): + queryset = models.Experiment.filter_manager.filter_qs() + serializer_class = serializers.ExperimentReadSerializer + filterset_class = filters.ExperimentFilter + filter_permissions = "experiment_upload__project" + + class CanonSiteConfView(ISPyBSafeQuerySet): queryset = models.CanonSiteConf.filter_manager.filter_qs().filter(superseded=False) serializer_class = serializers.CanonSiteConfReadSerializer