Added source files to database and API (issue 1497)

api/urls.py

@@ -113,6 +113,7 @@
     "site_observations", viewer_views.SiteObservationView, basename='site_observations'
 )
 router.register("canon_sites", viewer_views.CanonSiteView, basename='canon_sites')
+router.register("experiments", viewer_views.ExperimentView, basename='experiments')
 router.register(
     "canon_site_confs", viewer_views.CanonSiteConfView, basename='canon_site_confs'
 )

viewer/filters.py

@@ -7,6 +7,7 @@
     CanonSite,
     CanonSiteConf,
     Compound,
+    Experiment,
     Pose,
     QuatAssembly,
     SiteObservation,
@@ -60,6 +61,12 @@ class Meta:
         fields = ("target",)
 
 
+class ExperimentFilter(TargetFilterMixin):
+    class Meta:
+        model = Experiment
+        fields = ("target",)
+
+
 class CanonSiteConfFilter(TargetFilterMixin):
     class Meta:
         model = CanonSiteConf

viewer/models.py

@@ -215,13 +215,17 @@ class Experiment(models.Model):
     pdb_info = models.FileField(
         upload_to="target_loader_data/", null=True, max_length=255
     )
+    pdb_info_source_file = models.TextField(null=True)
     mtz_info = models.FileField(
         upload_to="target_loader_data/", null=True, max_length=255
     )
+    mtz_info_source_file = models.TextField(null=True)
     cif_info = models.FileField(
         upload_to="target_loader_data/", null=True, max_length=255
     )
+    cif_info_source_file = models.TextField(null=True)
     map_info = ArrayField(models.FileField(max_length=255), null=True)
+    map_info_source_files = ArrayField(models.TextField(null=True), null=True)
     type = models.PositiveSmallIntegerField(null=True)
     pdb_sha256 = models.TextField(null=True)
     prefix_tooltip = models.TextField(null=True)

viewer/serializers.py

@@ -931,6 +931,12 @@ class Meta:
         fields = '__all__'
 
 
+class ExperimentReadSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.Experiment
+        fields = '__all__'
+
+
 class CanonSiteConfReadSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.CanonSiteConf

viewer/target_loader.py

@@ -523,7 +523,7 @@ def validate_map_files(
         obj_identifier: str,
         file_struct: list,
         validate_files: bool = True,
-    ) -> list[str]:
+    ) -> tuple[list[str], list[str]]:
         """Validate list of panddas event files.
 
         Special case of file validation, too complex to squeeze into
@@ -533,17 +533,20 @@ def validate_map_files(
         def logfunc(_, message):
             self.report.log(logging.WARNING, message)
 
-        result = []
+        paths = []
+        source_files = []
         for item in file_struct:
             fname, file_hash = self._check_file(item, obj_identifier, key, logfunc)
+            source_file = item.get("source_file", None)
             if not fname:
                 continue
 
             if validate_files:
                 self._check_file_hash(obj_identifier, key, fname, file_hash, logfunc)
-            result.append(fname)
+            paths.append(fname)
+            source_files.append(source_file)
 
-        return result
+        return paths, source_files
 
     def validate_files(
         self,
@@ -552,7 +555,7 @@ def validate_files(
         required: Iterable[str] = (),
         recommended: Iterable[str] = (),
         validate_files: bool = True,
-    ) -> list[str | None]:
+    ) -> list[tuple[str | None, str | None]]:
         """Check if file exists and if sha256 hash matches (if given).
 
         file struct can come in 2 configurations:
@@ -594,7 +597,7 @@ def logfunc(key, message):
                 # schema isn't looking for this file, ignore
                 continue
 
-            filename, file_hash = None, None
+            filename, file_hash, source_file = None, None, None
 
             # sort out the filename
             if isinstance(value, dict):
@@ -604,6 +607,8 @@ def logfunc(key, message):
                 if not filename:
                     continue
 
+                source_file = value.get("source_file", None)
+
                 if validate_files:
                     self._check_file_hash(
                         obj_identifier, key, filename, file_hash, logfunc
@@ -627,7 +632,7 @@ def logfunc(key, message):
         files = []
         for f in list(required) + list(recommended):
             try:
-                files.append(result[f])
+                files.append((result[f], source_file))
             except KeyError:
                 logfunc(
                     f,
@@ -637,7 +642,7 @@ def logfunc(key, message):
                         METADATA_FILE,
                     ),
                 )
-                files.append(None)  # type: ignore [arg-type]
+                files.append((None, None))  # type: ignore [arg-type]
 
         logger.debug("Returning files: %s", files)
 
@@ -753,9 +758,9 @@ def process_experiment(
         )
 
         (  # pylint: disable=unbalanced-tuple-unpacking
-            pdb_info,
-            mtz_info,
-            cif_info,
+            pdb_info_t,
+            mtz_info_t,
+            cif_info_t,
         ) = self.validate_files(
             obj_identifier=experiment_name,
             file_struct=data["crystallographic_files"],
@@ -767,12 +772,16 @@ def process_experiment(
             validate_files=validate_files,
         )
 
+        pdb_info, pdb_info_source_file = pdb_info_t
+        mtz_info, mtz_info_source_file = mtz_info_t
+        cif_info, cif_info_source_file = cif_info_t
+
         try:
             event_files = data["crystallographic_files"]["ligand_binding_events"]
         except KeyError:
             event_files = []
 
-        map_info_files = self.validate_map_files(
+        map_info_files, map_info_source_files = self.validate_map_files(
             key="ligand_binding_events",
             obj_identifier=experiment_name,
             file_struct=event_files,
@@ -836,7 +845,11 @@ def process_experiment(
             "pdb_info": str(self._get_final_path(pdb_info)),
             "mtz_info": str(self._get_final_path(mtz_info)),
             "cif_info": str(self._get_final_path(cif_info)),
+            "pdb_info_source_file": pdb_info_source_file,
+            "mtz_info_source_file": mtz_info_source_file,
+            "cif_info_source_file": cif_info_source_file,
             "map_info": map_info_paths,
+            "map_info_source_files": map_info_source_files,
             "prefix_tooltip": prefix_tooltip,
             "code_prefix": code_prefix,
             # this doesn't seem to be present
@@ -1351,17 +1364,17 @@ def process_site_observation(
         xtalform_site = xtalform_sites[v_key]
 
         (  # pylint: disable=unbalanced-tuple-unpacking
-            bound_file,
-            apo_solv_file,
-            apo_desolv_file,
-            apo_file,
-            artefacts_file,
-            sigmaa_file,
-            diff_file,
-            event_file,
-            ligand_pdb,
-            ligand_mol,
-            ligand_smiles,
+            bound_file_t,
+            apo_solv_file_t,
+            apo_desolv_file_t,
+            apo_file_t,
+            artefacts_file_t,
+            sigmaa_file_t,
+            diff_file_t,
+            event_file_t,
+            ligand_pdb_t,
+            ligand_mol_t,
+            ligand_smiles_t,
         ) = self.validate_files(
             obj_identifier=experiment_id,
             file_struct=data,
@@ -1383,6 +1396,18 @@ def process_site_observation(
             validate_files=validate_files,
         )
 
+        bound_file = bound_file_t[0]
+        apo_solv_file = apo_solv_file_t[0]
+        apo_desolv_file = apo_desolv_file_t[0]
+        apo_file = apo_file_t[0]
+        artefacts_file = artefacts_file_t[0]
+        sigmaa_file = sigmaa_file_t[0]
+        diff_file = diff_file_t[0]
+        event_file = event_file_t[0]
+        ligand_pdb = ligand_pdb_t[0]
+        ligand_mol = ligand_mol_t[0]
+        ligand_smiles = ligand_smiles_t[0]
+
         fields = {
             # Code for this protein (e.g. Mpro_Nterm-x0029_A_501_0)
             "longcode": longcode,
@@ -1548,6 +1573,10 @@ def process_bundle(self):
             required=(TRANS_NEIGHBOURHOOD, TRANS_CONF_SITE, TRANS_REF_STRUCT),
         )
 
+        trans_neighbourhood = trans_neighbourhood[0]
+        trans_conf_site = trans_conf_site[0]
+        trans_ref_struct = trans_ref_struct[0]
+
         self.experiment_upload.project = self.project
         self.experiment_upload.target = self.target
         self.experiment_upload.committer = committer

viewer/views.py

@@ -1741,6 +1741,13 @@ class CanonSiteView(ISPyBSafeQuerySet):
     )
 
 
+class ExperimentView(ISPyBSafeQuerySet):
+    queryset = models.Experiment.filter_manager.filter_qs()
+    serializer_class = serializers.ExperimentReadSerializer
+    filterset_class = filters.ExperimentFilter
+    filter_permissions = "experiment_upload__project"
+
+
 class CanonSiteConfView(ISPyBSafeQuerySet):
     queryset = models.CanonSiteConf.filter_manager.filter_qs().filter(superseded=False)
     serializer_class = serializers.CanonSiteConfReadSerializer