diff --git a/.github/workflows/build-staging.yaml b/.github/workflows/build-staging.yaml index 16c1eddf..c5e0197c 100644 --- a/.github/workflows/build-staging.yaml +++ b/.github/workflows/build-staging.yaml @@ -64,7 +64,7 @@ env: # For Jobs conditional on the presence of a secret see this Gist... # https://gist.github.com/jonico/24ffebee6d2fa2e679389fac8aef50a3 BE_NAMESPACE: xchem - FE_BRANCH: production + FE_BRANCH: staging FE_NAMESPACE: xchem STACK_BRANCH: master STACK_GITHUB_NAMESPACE: xchem diff --git a/.gitignore b/.gitignore index 502a5088..9bf99724 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ static/ .idea/ +.vscode/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d7b95e8a..9f282b84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ --- -minimum_pre_commit_version: 2.20.0 +minimum_pre_commit_version: 2.21.0 exclude: ^(design_docs|doc_templates|docs|tests|) repos: @@ -40,8 +40,13 @@ repos: # the user has done 90% of the lint checks before the code # hits the server. - repo: https://github.com/pycqa/pylint - rev: v2.14.4 + rev: v2.16.1 hooks: - id: pylint args: - - --disable=import-error + - --django-settings-module + - fragalysis.settings + - --load-plugins + - pylint_django + - --load-plugins + - pylint_django.checkers.migrations diff --git a/.pylintrc b/.pylintrc index 57b0e145..e207519c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -5,6 +5,10 @@ # # C : Convention warnings # R : Refactoring warnings -disable = C, R, +disable = C, F, R, + bare-except, + broad-exception-caught, + broad-exception-raised, + fixme, import-error, - django-not-configured + unused-variable diff --git a/README.md b/README.md index d4fd1f13..9de6e718 100644 --- a/README.md +++ b/README.md @@ -94,14 +94,18 @@ The best approach is to spin-up the development stack (locally) using to make new migrations called "add_job_request_start_and_finish_times" for the viewer's models run the following: - +> Before starting postgres, if you need to, remove any pre-existing Db (if one exists) + with `rm -rf data` + docker-compose up -d docker-compose exec stack bash -Then from within the stack... +Then from within the stack make the migrations. Here we're migrating the `viewer` +application... python manage.py makemigrations viewer --name "add_job_request_start_and_finish_times" -Exit the container and tear-down the deployemnt: - +Exit the container and tear-down the deployment: - docker-compose down @@ -125,7 +129,7 @@ Now the project's rules will run on every commit. Start `Fragalysis stack` (All infrastructure - databases + populating data) ``` -docker-compose -f docker-compose.dev.yml up -d +docker-compose up -d ``` `Please wait, it takes a minute until all containers are fully started.` @@ -135,7 +139,7 @@ Test if we are running at [http://localhost:8080](http://localhost:8080) If needed stop containers ``` -docker-compose -f docker-compose.dev.yml down +docker-compose down ``` diff --git a/api/security.py b/api/security.py index a6ca7466..6c399a3d 100644 --- a/api/security.py +++ b/api/security.py @@ -1,8 +1,11 @@ +import logging import os import time + from wsgiref.util import FileWrapper from django.http import Http404 from django.http import HttpResponse +from django.db.models import Q from ispyb.connector.mysqlsp.main import ISPyBMySQLSPConnector as Connector from ispyb.connector.mysqlsp.main import ISPyBNoResultException from rest_framework import viewsets @@ -10,6 +13,8 @@ from viewer.models import Project +logger = logging.getLogger(__name__) + USER_LIST_DICT = {} connector = os.environ.get('SECURITY_CONNECTOR', 'ispyb') @@ -54,9 +59,17 @@ def get_remote_conn(): # Assume the credentials are invalid if there is no host. # If a host is not defined other properties are useless. if not ispyb_credentials["host"]: + logger.debug("No ISPyB host - cannot return a connector") return None - conn = SSHConnector(**ispyb_credentials) + # Try to get an SSH connection (aware that it might fail) + conn = None + try: + conn = SSHConnector(**ispyb_credentials) + except: + logger.info("ispyb_credentials=%s", ispyb_credentials) + logger.exception("Exception creating SSHConnector") + return conn @@ -76,7 +89,13 @@ def get_conn(): if not credentials["host"]: return None - conn = Connector(**credentials) + conn = None + try: + conn = Connector(**credentials) + except: + logger.info("credentials=%s", credentials) + logger.exception("Exception creating Connector") + return conn @@ -87,34 +106,53 @@ def get_queryset(self): Optionally restricts the returned purchases to a given proposals """ # The list of proposals this user can have - proposal_list = self.get_proposals_for_user() + proposal_list = self.get_proposals_for_user(self.request.user) # Add in the ones everyone has access to - proposal_list.extend(self.get_open_proposals()) - # Must have a directy foreign key (project_id) for it to work - filter_dict = self.get_filter_dict(proposal_list) - return self.queryset.filter(**filter_dict).distinct() + # (unless we already have it) + for open_proposal in self.get_open_proposals(): + if open_proposal not in proposal_list: + proposal_list.append(open_proposal) + + logger.debug('is_authenticated=%s, proposal_list=%s', + self.request.user.is_authenticated, proposal_list) + + # Must have a foreign key to a Project for this filter to work. + # get_q_filter() returns a Q expression for filtering + q_filter = self.get_q_filter(proposal_list) + return self.queryset.filter(q_filter).distinct() def get_open_proposals(self): """ - Returns the list of proposals anybody can access - :return: + Returns the list of proposals anybody can access. + This function is deprecated, instead we should move to the 'open_to_public' + field rather than using a built-in list of Projects. + We still add "OPEN" to the list for legacy testing. """ if os.environ.get("TEST_SECURITY_FLAG", False): - return ["OPEN", "private_dummy_project"] + return ["lb00000", "OPEN", "private_dummy_project"] else: - return ["OPEN", "lb27156"] + # A list of well-known (built-in) public Projects (Proposals/Visits) + return ["lb00000", "OPEN", "lb27156"] def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user if user.pk is None: + logger.warning("user.pk is None") return [] else: - return list( + prop_ids = list( Project.objects.filter(user_id=user.pk).values_list("title", flat=True) ) + logger.debug("Got %s proposals: %s", len(prop_ids), prop_ids) + return prop_ids def needs_updating(self, user): + """Returns true of the data collected for a user is out of date. + It's simple, we just record the last collected timestamp and consider it + 'out of date' (i.e. more than an hour old). + """ global USER_LIST_DICT + update_window = 3600 if user.username not in USER_LIST_DICT: USER_LIST_DICT[user.username] = {"RESULTS": [], "TIMESTAMP": 0} @@ -125,6 +163,7 @@ def needs_updating(self, user): return False def run_query_with_connector(self, conn, user): + core = conn.core try: rs = core.retrieve_sessions_for_person_login(user.username) @@ -139,43 +178,115 @@ def run_query_with_connector(self, conn, user): def get_proposals_for_user_from_ispyb(self, user): # First check if it's updated in the past 1 hour global USER_LIST_DICT - if self.needs_updating(user): + + needs_updating = self.needs_updating(user) + logger.debug("user=%s needs_updating=%s", user.username, needs_updating) + + if needs_updating: conn = None if connector == 'ispyb': conn = get_conn() if connector == 'ssh_ispyb': conn = get_remote_conn() - # If there is no connection (ISpyB credentials may be missing) + # If there is no connection (ISPyB credentials may be missing) # then there's nothing we can do except return an empty list. + # Otherwise run a query for the user. if conn is None: + logger.warning("Failed to get ISPyB connector") return [] - rs = self.run_query_with_connector(conn=conn, user=user) - - visit_ids = list(set([ - str(x["proposalNumber"]) + "-" + str(x["sessionNumber"]) for x in rs - ])) - prop_ids = list(set([str(x["proposalNumber"]) for x in rs])) - prop_ids.extend(visit_ids) - USER_LIST_DICT[user.username]["RESULTS"] = prop_ids - return prop_ids - else: + logger.debug("Connector query rs=%s", rs) + + # Typically you'll find the following fields in each item + # in the rs response; - + # + # 'id': 0000000, + # 'proposalId': 00000, + # 'startDate': datetime.datetime(2022, 12, 1, 15, 56, 30) + # 'endDate': datetime.datetime(2022, 12, 3, 18, 34, 9) + # 'beamline': 'i00-0' + # 'proposalCode': 'lb' + # 'proposalNumber': '12345' + # 'sessionNumber': 1 + # 'comments': None + # 'personRoleOnSession': 'Data Access' + # 'personRemoteOnSession': 1 + # + # Iterate through the response and return the 'proposalNumber' (proposals) + # and one with the 'proposalNumber' and 'sessionNumber' (visits), each + # prefixed by the `proposalCode` (if present). + # + # Codes are expected to consist of 2 letters. + # Typically: lb, mx, nt, nr, bi + # + # These strings should correspond to a title value in a Project record. + # and should get this sort of list: - + # + # ["lb12345", "lb12345-1"] + # -- - + # | ----- | + # Code | Session + # Proposal + prop_id_set = set() + for record in rs: + pc_str = "" + if "proposalCode" in record and record["proposalCode"]: + pc_str = f'{record["proposalCode"]}' + pn_str = f'{record["proposalNumber"]}' + sn_str = f'{record["sessionNumber"]}' + proposal_str = f'{pc_str}{pn_str}' + proposal_visit_str = f'{proposal_str}-{sn_str}' + prop_id_set.update([proposal_str, proposal_visit_str]) + + # Always display the collected results for the user. + # These will be cached. + logger.info("Got %s proposals (%s): %s", + len(prop_id_set), user.username, prop_id_set) + + # Cache the result and return the result for the user + USER_LIST_DICT[user.username]["RESULTS"] = list(prop_id_set) return USER_LIST_DICT[user.username]["RESULTS"] + else: + # Return the previous query (cached for an hour) + cached_prop_ids = USER_LIST_DICT[user.username]["RESULTS"] + logger.debug("Got %s cached proposals: %s", len(cached_prop_ids), cached_prop_ids) + return cached_prop_ids - def get_proposals_for_user(self): - user = self.request.user + def get_proposals_for_user(self, user): + """Returns a list of proposals (public and private) that the user has access to. + """ + assert user + ispyb_user = os.environ.get("ISPYB_USER") + logger.debug("ispyb_user=%s", ispyb_user) if ispyb_user: + logger.debug("user.is_authenticated=%s", user.is_authenticated) if user.is_authenticated: return self.get_proposals_for_user_from_ispyb(user) else: + logger.debug("Got no proposals") return [] else: return self.get_proposals_for_user_from_django(user) - def get_filter_dict(self, proposal_list): - return {self.filter_permissions + "__title__in": proposal_list} + def get_q_filter(self, proposal_list): + """Returns a Q expression representing a (potentially complex) table filter. + """ + if self.filter_permissions: + # Q-filter is based on the filter_permissions string + # whether the resultant Project title in the proposal list + # OR where the Project is 'open_to_public' + return Q(**{self.filter_permissions + "__title__in": proposal_list}) |\ + Q(**{self.filter_permissions + "__open_to_public": True}) + else: + # No filter permission? + # Assume this QuerySet is used for the Project model. + # Added during 937 development (Access Control). + # + # Q-filter is based on the Project title being in the proposal list + # OR where the Project is 'open_to_public' + return Q(title__in=proposal_list) | Q(open_to_public=True) class ISpyBSafeStaticFiles: diff --git a/api/urls.py b/api/urls.py index 3ad51f3e..daa8886f 100644 --- a/api/urls.py +++ b/api/urls.py @@ -15,6 +15,7 @@ router.register(r"compounds", viewer_views.CompoundView) router.register(r"targets", viewer_views.TargetView, "targets") router.register(r"proteins", viewer_views.ProteinView) +router.register(r"projects", viewer_views.ProjectView) router.register(r"session-projects", viewer_views.SessionProjectsView) router.register(r"snapshots", viewer_views.SnapshotsView) router.register(r"action-type", viewer_views.ActionTypeView) @@ -84,7 +85,6 @@ # Squonk Jobs router.register(r"job_file_transfer", viewer_views.JobFileTransferView, basename='job_file_transfer') -router.register(r"job_request", viewer_views.JobRequestView, basename='job_request') router.register(r"job_callback", viewer_views.JobCallBackView, basename='job_callback') router.register(r"job_config", viewer_views.JobConfigView, basename='job_config') @@ -105,4 +105,6 @@ def schema_view(request): url(r"^", include(router.urls)), url(r"^auth$", drf_views.obtain_auth_token, name="auth"), url(r"^swagger$", schema_view), + + url(r"job_request", viewer_views.JobRequestView.as_view(), name="job_request"), ] diff --git a/api/utils.py b/api/utils.py index 4c358e7c..6b338bd5 100644 --- a/api/utils.py +++ b/api/utils.py @@ -156,8 +156,10 @@ def draw_mol(smiles, height=49, width=150, bondWidth=1, scaling=1.0, img_type=No # If you want to influence this use the scaling parameter. x, y = calc_bounds(conformer) dim_x, dim_y = calc_dims(x, y) - scale_x = width / dim_x - scale_y = height / dim_y + # Protect scaling from Div0. + # Scale factors are 0 if the corresponding dimension is not +ve, non=zero. + scale_x = width / dim_x if dim_x > 0 else 0 + scale_y = height / dim_y if dim_y > 0 else 0 scale = min(scale_x, scale_y) font = max(round(scale * scaling), 6) @@ -319,3 +321,37 @@ def mol_view(request): return get_params(smiles, request) else: return HttpResponse("Please insert SMILES") + + +def pretty_request(request, *, tag='', print_body=False): + """A simple function to return a Django request as nicely formatted string.""" + headers = '' + if request.headers: + for key, value in request.headers.items(): + headers += f'{key}: {value}\n' + + tag_text = f'{tag}\n' if tag else '' + + user_text = 'User: ' + if request.user: + user_text += str(request.user) + else: + user_text += '(-)' + + # Load the body but cater for problems, like + # django.http.request.RawPostDataException: + # You cannot access body after reading from request's data stream + body = None + if print_body: + try: + body = request.body + except Exception: + pass + + return f'{tag_text}' \ + '+ REQUEST BEGIN\n' \ + f'{user_text}\n' \ + f'{request.method} HTTP/1.1\n' \ + f'{headers}\n\n' \ + f'{body}\n' \ + '- REQUEST END' diff --git a/build-requirements.txt b/build-requirements.txt index 7413b6a9..93f5d06b 100644 --- a/build-requirements.txt +++ b/build-requirements.txt @@ -1,5 +1,7 @@ # Requirements specifically used to build/test the backend. # These are not rquirements needed by the backend in order for it to run. -pylint -pylint-django -pre-commit == 2.20.0 +pylint == 2.16.1 +pylint-django == 2.5.3 +pre-commit == 2.21.0 + +Django==3.1.14 diff --git a/design_docs/squonk2-access-control.puml b/design_docs/squonk2-access-control.puml new file mode 100644 index 00000000..b5bf6634 --- /dev/null +++ b/design_docs/squonk2-access-control.puml @@ -0,0 +1,73 @@ +@startuml +' hide the spot +hide circle + +' avoid problems with angled crows feet +skinparam linetype ortho + +entity Squonk2Org #yellowgreen { + *uuid : string[40] + *name : string[80] + *as_url : url + *as_version : string + -- + id : number <> +} + +entity Squonk2Unit #yellowgreen { + *uuid : string[41] + *name : string[80] + -- + id : number <> + project : number <> + organisation : number <> +} + +entity Squonk2Project #yellowgreen { + *uuid : string[44] + *name : string[80] + *product_uuid : string[44] + -- + id : number <> + squonk2_unit_id : number <> + user_id : number <> + target_id : number <> +} + +note right of Squonk2Org + New tables in green +end note + +entity Project { + title : string + -- + id : number <> +} + +note left of Project + The project title + (a hyphenated Proposal/Session string) + is used to create a unit. + Each proposal has its own unit. +end note + +entity User { + -- + *id : number <> +} + +entity SessionProject { + *title : string[200] + *target : number <> + -- + id : number <> +} + +Squonk2Org ||..|{ Squonk2Unit +Squonk2Unit ||..|{ Squonk2Project +Squonk2Unit ||..|| Project +Squonk2Project ||..|| User +Project }o..o{ User +Squonk2Project ||..|| SessionProject + +@enduml diff --git a/docker-compose.yml b/docker-compose.yml index 0c4c284f..6ecbaeb5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,14 @@ --- + +# You typically create .env file to populate the +# sensitive variables for the stack deployment. +# Then bring the containers up with: - +# docker-compose up -d +# Then enter the stack container with: - +# docker-compose exec stack bash +# Where you should then find the back end at http://localhost:8080/api/ +# and logs in /code/logs/backend.log + version: '3' services: @@ -8,14 +18,19 @@ services: image: postgres:12.2 container_name: database volumes: - - ../data/postgre/data:/var/lib/postgresql/data + - ../data/postgresql/data:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: fragalysis - POSTGRES_USER: fragalysis POSTGRES_DB: frag PGDATA: /var/lib/postgresql/data/pgdata ports: - "5432:5432" + healthcheck: + test: pg_isready -U postgres -d frag + interval: 10s + timeout: 2s + retries: 5 + start_period: 10s # The graph graph: @@ -35,6 +50,12 @@ services: environment: - NEO4J_AUTH=none - NEO4J_dbms_memory_pagecache_size=4G + healthcheck: + test: wget http://localhost:7474 || exit 1 + interval: 10s + timeout: 10s + retries: 20 + start_period: 10s # The stack (backend) stack: @@ -50,20 +71,39 @@ services: - .:/code/ environment: POSTGRESQL_DATABASE: frag - POSTGRESQL_USER: fragalysis + POSTGRESQL_USER: postgres POSTGRESQL_PASSWORD: fragalysis POSTGRESQL_HOST: database POSTGRESQL_PORT: 5432 + # Logging level + LOGGING_FRAMEWORK_ROOT_LEVEL: ${LOGGING_FRAMEWORK_ROOT_LEVEL:-INFO} + # Celery tasks run synchronously? + CELERY_TASK_ALWAYS_EAGER: 'True' # Default Debug to true for local development - DEBUG_FRAGALYSIS: 'True' - DISABLE_LOGGING_FRAMEWORK: 'No' OIDC_RP_CLIENT_SECRET: 'c6245428-04c7-466f-9c4f-58c340e981c2' - SQUONK2_UI_URL: https://data-manager-ui.xchem-dev.diamond.ac.uk/data-manager-ui - SQUONK2_DMAPI_URL: https://data-manager.xchem-dev.diamond.ac.uk/data-manager-api - # Celery tasks run synchronously? -# CELERY_TASK_ALWAYS_EAGER: 'True' + SQUONK2_VERIFY_CERTIFICATES: 'No' + SQUONK2_UNIT_BILLING_DAY: 3 + SQUONK2_PRODUCT_FLAVOUR: BRONZE + SQUONK2_SLUG: fs-local + # The following are normally populated via a local '.env' file + # that is not part of the repo, it is a file you create + # and is used by docker-compose if it finds it. + OIDC_KEYCLOAK_REALM: ${OIDC_KEYCLOAK_REALM} + OIDC_AS_CLIENT_ID: ${OIDC_AS_CLIENT_ID} + OIDC_DM_CLIENT_ID: ${OIDC_DM_CLIENT_ID} + SQUONK2_ORG_OWNER: ${SQUONK2_ORG_OWNER} + SQUONK2_ORG_OWNER_PASSWORD: ${SQUONK2_ORG_OWNER_PASSWORD} + SQUONK2_ORG_UUID: ${SQUONK2_ORG_UUID} + SQUONK2_UI_URL: ${SQUONK2_UI_URL} + SQUONK2_DMAPI_URL: ${SQUONK2_DMAPI_URL} + SQUONK2_ASAPI_URL: ${SQUONK2_ASAPI_URL} + DUMMY_TAS: ${DUMMY_TAS} + DUMMY_USER: ${DUMMY_USER} + DUMMY_TARGET_TITLE: ${DUMMY_TARGET_TITLE} ports: - "8080:80" depends_on: - - database - - graph + database: + condition: service_healthy + graph: + condition: service_healthy diff --git a/docs/source/API/download.rst b/docs/source/API/download.rst index fe5cce14..01e4bbd2 100644 --- a/docs/source/API/download.rst +++ b/docs/source/API/download.rst @@ -10,10 +10,8 @@ Fragalysis provides functionality to flexibly download subsets of data as follow - **Subset of Computed Set Data** - Constructs a csv file for download based on a dictionary constructed in the react front end from the computed sets. - - Views ---------- - +Views +----- .. autoclass:: viewer.views.DownloadStructures :members: diff --git a/docs/source/computational_data/views.rst b/docs/source/computational_data/views.rst index cbee16ee..0cffbccf 100644 --- a/docs/source/computational_data/views.rst +++ b/docs/source/computational_data/views.rst @@ -19,9 +19,6 @@ a version of the standard DRF (see :ref:`RESTful API (Views) `) :code is used to generate the 2D image shown on each molecule in the RHS computed set tab using :code:`viewer.views.img_from_smiles`. The scores for each molecule are displayed on the respective molecule card. -- :code:`viewer.views.cset_key`: This view is used to generate a computed set upload key at :code:`/viewer/cset_key` - , and email it to the user to allow them to upload new computed sets at :code:`/viewer/upload_cset` - - :code:`viewer.views.UploadCSet`: This view is used to generate the form found at :code:`/viewer/upload_cset`, and to pass the values to the celery tasks controlled through :code:`viewer.views.ValidateTaskView` and :code:`viewer.views.UploadTaskView` that validate and process the uploaded data, saving it into the relevant models @@ -52,9 +49,6 @@ View details .. autoclass:: viewer.views.ComputedMolAndScoreView :members: -.. autoclass:: viewer.views.cset_key - :members: - .. autoclass:: viewer.views.UploadCSet :members: @@ -72,4 +66,3 @@ View details .. autoclass:: viewer.views.pset_download :members: - diff --git a/docs/source/index.rst b/docs/source/index.rst index dad34f66..91becaa3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,6 +25,8 @@ Contents computational_data/tasks API/media API/download + vectors + squonk/integration Indices and tables diff --git a/docs/source/squonk/integration.rst b/docs/source/squonk/integration.rst new file mode 100644 index 00000000..23dd7315 --- /dev/null +++ b/docs/source/squonk/integration.rst @@ -0,0 +1,158 @@ +Squonk Integration +================== + +The `Squonk`_ **Data Manager** and associated services, developed by `Informatics Matters`_, +provides a novel, easy to use, web based, data centric workflow environment in which +scientists can execute scientific workflows using open source and commercial tools from +multiple sources such as RDKit, Chemistry Development Kit (CDK), ChemAxon. + +A RESTful API is also available for accessing the data and services **Squonk** provides. + +Fragalysis has been adapted so that it can utilise **Squonk** services via its API +and Graphical User Interface. + +Squonk Installation +------------------- + +Rather than use the commercial **Squonk** service a *custom* **Squonk** installation +is typically provided, running in the same cluster as the Fragalysis Stack that +you wish to use. The installation will consist of the **Squonk** *Data Manager* +it accounting service, the *account server*. The *Data Manager* and *Account Server* +will be configured to use the keycloak authentication server also used by Fragalysis. + +The administrator of these applications will need to ensure the following: - + +Fragalysis Squonk User +^^^^^^^^^^^^^^^^^^^^^^ + +An administrative user is made available for use exclusively by Fragalysis. This is +the user Fragalysis will use to automate the orchestration of objects in **Squonk** - +the creation of *Units*, *Products*, *Projects* and Jobs (*Instances*). The user name +is typically `fragalysis`. + +.. note:: + If more than one Fragalysis Stack is configured to use one **Squonk** installation + the user can be shared between them - you don't necessarily need a separate + `fragalysis` user for each Fragalysis Stack. + +Fragalysis Squonk Organisation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An *Organisation* needs to be created that will be the root of all objects created +by Fragalysis (its *Units*, *Products* and *Projects*). You don't need to know +how these objects relate to Fragalysis activity, only that an *Organisation* must +be provisioned. + +.. note:: + The Organisation is Stack-specific - **EVERY** Fragalysis Stack must be assigned + a unique organisation. The *Staging* and *Production* stacks currently share + access to One **Squonk**, and they each do this using separate *Organisations*. + +Squonk User Roles +^^^^^^^^^^^^^^^^^ + +Users that use Fragalysis, and also need access to **Squonk** services, +will need to be assigned the ``data-manager-user`` and ``account-server-user`` **Roles** +in the corresponding keycloak service. + +Squonk Integration +------------------ + +With the **Squonk** installation in place, the Fragalysis Stack needs be configured +using the following Ansible playbook variables, which translate to environment variables +available to the Fragalysis Stack container: - + +.. list-table:: Stack Playbook Variables + :widths: 30 70 + :header-rows: 1 + + * - Variable + - Description + * - ``stack_oidc_as_client_id`` + - The client ID of the Account Server in the keycloak authentication server. + * - ``stack_oidc_dm_client_id`` + - The client ID of the Data Manager in the keycloak authentication server. + * - ``stack_squonk2_ui_url`` + - The URL of the installed Squonk UI, typically ``https://squonk.example.com/data-manager-ui`` + * - ``stack_squonk2_dmapi_url`` + - The URL of the installed Squonk Data Manager API, typically ``https://squonk.example.com/data-manager-api`` + * - ``stack_squonk2_asapi_url`` + - The URL of the installed Squonk Account Server API, typically ``https://squonk.example.com/account-server-api`` + * - ``stack_squonk2_org_owner`` + - The name of the Fragalysis user that will be used to create objects in Squonk. + * - ``stack_squonk2_org_owner_password`` + - The password of the Fragalysis user that will be used to create objects in Squonk. + * - ``stack_squonk2_org_uuid`` + - The UUID of the Squonk Organisation under which *Units*, *Products* and *Projects* + will be created. This must be unique for each Fragalysis Stack. + * - ``stack_squonk2_product_flavour`` + - The *flavour* of the Squonk *Products* that will be created. + This must be one of ``BRONZE``, ``SILVER`` or ``GOLD``. + * - ``stack_squonk2_slug`` + - The *slug* used to create objects in Squonk *Product*. This is used to + identify the Fragalysis Stack that created the object and must be + unique for each Fragalysis Stack. It is a short string (up to 10 characters). + The staging stack might use ``staging`` and the production stack might use + ``production``. + * - ``stack_squonk2_unit_billing_day`` + - The day of the month on which the *Units* will be billed. This must be + an integer between 1 and 28. + +.. warning:: + The ``stack_squonk2_org_uuid``, which must be unique for each Fragalysis Stack, + cannot currently be changed once the stack has been launched. It is the + responsibility of the administrator to ensure that the UUID is unique and the + variable is correctly set prior to launching the stack. + +Squonk Model +------------ + +The following Fragalysis ``viewer.models`` are used by the Fragalysis Stack to manage +access to and record the **Squonk** objects that are created: - + +.. autoclass:: viewer.models.Squonk2Org + :members: + +.. autoclass:: viewer.models.Squonk2Unit + :members: + +.. autoclass:: viewer.models.Squonk2Project + :members: + +.. autoclass:: viewer.models.JobFileTransfer + :members: + +.. autoclass:: viewer.models.JobRequest + :members: + +Squonk Views +------------ + +.. autoclass:: viewer.views.JobConfigView + :members: + +.. autoclass:: viewer.views.JobFileTransferView + :members: + +.. autoclass:: viewer.views.JobRequestView + :members: + +.. autoclass:: viewer.views.JobCallBackView + :members: + +.. autoclass:: viewer.views.JobAccessView + :members: + +Squonk2Agent Class +------------------ + +The main interactions with **Squonk** are handled by the ``Squonk2Agent`` class +in the ``viewer`` package and the `Squonk2 Python Client`_, which provides API access +to **Squonk** through its ``DmApi`` and ``AsApi`` classes. + +.. autoclass:: viewer.squonk2_agent.Squonk2Agent + :members: + +.. _Informatics Matters: https://www.informaticsmatters.com/ +.. _Squonk: https://squonk.it/ +.. _Squonk2 Python Client: https://github.com/InformaticsMatters/squonk2-python-client diff --git a/docs/source/vector-mols.png b/docs/source/vector-mols.png new file mode 100644 index 00000000..1b8fe6b3 Binary files /dev/null and b/docs/source/vector-mols.png differ diff --git a/docs/source/vectors.rst b/docs/source/vectors.rst new file mode 100644 index 00000000..1a5e8208 --- /dev/null +++ b/docs/source/vectors.rst @@ -0,0 +1,61 @@ +Fragment Network Vectors +======================== + +This document tries to describe how the fragment network vectors are handled. + +Vector Generation +----------------- + +Vector data is generated when a target is uploaded. +The entrypoint is the ``analyse_target()`` function in ``viewer/target_set_upload.py``. +Ultimately it’s a call to ``get_vectors()`` that actually creates the vectors and 3d records. + +There are two relevant database tables, ``hypothesis_vector`` and ``hypothesis_vector3d``. +The first contains the definition of the vectors (e.g. the location of the vector) with respect to the ``viewer_compound`` table. +The second contains the 3D coordinates (start an end point) of the actual 3D vector related to the ``viewer_molecule`` table. + +The ``get_vectors()`` function uses the ``frag.network.decorate.get_3d_vects_for_mol()`` from the +fragalysis repo (https://github.com/xchem/fragalysis) repo to analyse the molecule and create the vector data. +The data is returned as a Python dictionary. For example, for this SMILES ``COc1ccc(Cc2cc(N)nc(N)n2)c(OC)c1OC`` +the following data is generated:: + + { + 'linkers': { + 'COc1ccc([Xe])c(OC)c1OC.Nc1cc([Xe])nc(N)n1': [(-1.3931, -0.3178, 1.6803), (-2.8284, 1.2604, 3.3541), (-3.3955, 0.5713, 2.3605), (-2.8284, 1.2604, 3.3541)] + }, + 'deletions': { + 'COc1ccc(Cc2cc(N)nc([Xe])n2)c(OC)c1OC': [(-2.8284, 1.2604, 3.3541), (-2.8284, 1.2604, 3.3541)], + 'COc1ccc(Cc2cc([Xe])nc(N)n2)c(OC)c1OC': [(-2.7309, -0.2154, 1.5116), (-2.8284, 1.2604, 3.3541)], + 'COc1ccc(Cc2cc(N)nc(N)n2)c([Xe])c1OC': [(-0.7313, 0.3607, 2.6834), (-2.8284, 1.2604, 3.3541)], + 'COc1ccc(Cc2cc(N)nc(N)n2)c(OC)c1[Xe]': [(-1.4984, 1.1631, 3.4943), (-2.8284, 1.2604, 3.3541)], + 'COc1c([Xe])ccc(Cc2cc(N)nc(N)n2)c1OC': [(-0.2665, 0.2739, -1.2642), (-2.8284, 1.2604, 3.3541)] + }, + 'ring': { + 'COc1ccc(C[Xe])c(OC)c1OC.N[Xe].N[Xe]': [(-3.3955, 0.5713, 2.3605), (-2.8284, 1.2604, 3.3541), (-2.8284, 1.2604, 3.3541), (-2.8284, 1.2604, 3.3541), (-2.7309, -0.2154, 1.5116), (-2.8284, 1.2604, 3.3541)], + 'CO[Xe].CO[Xe].CO[Xe].Nc1cc(C[Xe])nc(N)n1': [(-0.7313, 0.3607, 2.6834), (-2.8284, 1.2604, 3.3541), (-1.4984, 1.1631, 3.4943), (-2.8284, 1.2604, 3.3541), (-0.2665, 0.2739, -1.2642), (-2.8284, 1.2604, 3.3541), (-1.3931, -0.3178, 1.6803), (-2.8284, 1.2604, 3.3541)] + }, + 'additions': { + 'COc1ccc(Cc2nc(N)nc(N)c2[Xe])c(OC)c1OC__0': [(-0.7313, 0.3607, 2.6834), (0.3550432234908941, 0.2643353596228167, 2.826830320575495)], + 'COc1cc([Xe])c(Cc2cc(N)nc(N)n2)c(OC)c1OC__0': [(-0.2665, 0.2739, -1.2642), (-1.3590949762544902, 0.3155256056977563, -1.3846305891945447)], + 'COc1c([Xe])cc(Cc2cc(N)nc(N)n2)c(OC)c1OC__0': [(0.5504, 0.9772, -2.1574), (0.09491642463792072, 1.5537821952831012, -2.9759888373645813)] + } + } + +Those molecules are as follows: + +.. image:: vector-mols.png + + +Vector Display +-------------- + +In the Fragalysis front end these vectors are displayed as arrows/cylinders emanating from the appropriate point +in the molecules. These imply a point of change in the molecule e.g. a potential substitution point. + +They are colour coded based on the occurrence of such changes in the fragment network database: +- Red means no examples found +- Yellow means 5 or fewer examples found +- Green means more than 5 examples found +TODO - confirm these numbers. + +TODO - describe how and when that data is generated \ No newline at end of file diff --git a/fix-target-file.py b/fix-target-file.py new file mode 100644 index 00000000..e69de29b diff --git a/fragalysis/settings.py b/fragalysis/settings.py index 847d1166..3be8c542 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -76,6 +76,11 @@ "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", "PAGE_SIZE": 5000, "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.QueryParameterVersioning", + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + 'mozilla_django_oidc.contrib.drf.OIDCAuthentication', + 'rest_framework.authentication.BasicAuthentication', + ], } # CELERY STUFF @@ -344,12 +349,6 @@ # dedicated Discourse server. DISCOURSE_DEV_POST_SUFFIX = os.environ.get("DISCOURSE_DEV_POST_SUFFIX", '') -# Squonk settings for API calls to Squonk Platform. -# The environment variable SQUONK2_DMAPI_URL -# is expected by the squonk2-client package. -SQUONK2_DMAPI_URL = os.environ.get('SQUONK2_DMAPI_URL') -SQUONK2_UI_URL = os.environ.get('SQUONK2_UI_URL') - SQUONK2_MEDIA_DIRECTORY = "fragalysis-files" SQUONK2_INSTANCE_API = "data-manager-ui/results/instance/" @@ -363,7 +362,7 @@ # (50Mi of logging in 10 files of 5M each), # with the rotating file handler typically used for everything. DISABLE_LOGGING_FRAMEWORK = True if os.environ.get("DISABLE_LOGGING_FRAMEWORK", "no").lower() in ["yes"] else False -LOGGING_FRAMEWORK_ROOT_LEVEL = os.environ.get("LOGGING_FRAMEWORK_ROOT_LEVEL", "INFO") +LOGGING_FRAMEWORK_ROOT_LEVEL = os.environ.get("LOGGING_FRAMEWORK_ROOT_LEVEL", "DEBUG") if not DISABLE_LOGGING_FRAMEWORK: LOGGING = { 'version': 1, @@ -371,7 +370,7 @@ 'formatters': { 'simple': { 'format': '%(asctime)s %(name)s.%(funcName)s():%(lineno)s %(levelname)s # %(message)s', - 'datefmt': '%Y-%m-%dT%H:%M:%S'}}, + 'datefmt': '%Y-%m-%dT%H:%M:%S%z'}}, 'handlers': { 'console': { 'level': 'DEBUG', @@ -385,13 +384,19 @@ 'filename': os.path.join(BASE_DIR, 'logs/backend.log'), 'formatter': 'simple'}}, 'loggers': { + 'api.security': { + 'level': 'INFO'}, 'asyncio': { 'level': 'WARNING'}, + 'celery': { + 'level': 'WARNING'}, 'django': { 'level': 'WARNING'}, 'mozilla_django_oidc': { - 'level': 'DEBUG'}, + 'level': 'WARNING'}, 'urllib3': { + 'level': 'WARNING'}, + 'paramiko': { 'level': 'WARNING'}}, 'root': { 'level': LOGGING_FRAMEWORK_ROOT_LEVEL, diff --git a/launch-stack.sh b/launch-stack.sh index 1a7559c4..ef1a29d3 100755 --- a/launch-stack.sh +++ b/launch-stack.sh @@ -22,7 +22,7 @@ printf "$script" | python manage.py shell touch /srv/logs/gunicorn.log touch /srv/logs/access.log touch /code/logs/logfile.log -tail -n 0 -f /srv/logs/*.log & +tail -n 0 -f /code/logs/*.log & echo "Starting Gunicorn...." cd /code gunicorn fragalysis.wsgi:application \ diff --git a/requirements.txt b/requirements.txt index 8d0249f2..731d7d55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,12 @@ coveralls==2.1.1 cryptography==3.2 decorator==4.4.2 deepdiff==5.8.1 + +# Dango is also referred to in the builtrequirements.txt file +# (for non-execution linting). +# Make sure they're the same! Django==3.1.14 + django-bootstrap3==14.1.0 django-cleanup==5.0.0 django-extensions==3.0.3 @@ -45,7 +50,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.3.0 +im-squonk2-client==1.22.0 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 @@ -101,6 +106,7 @@ Rx==1.6.1 scandir==1.10.0 SecretStorage==3.1.2 sentry-sdk==0.16.3 +shortuuid==1.0.11 simplegeneric==0.8.1 simplejson==3.17.2 singledispatch==3.4.0.3 @@ -121,8 +127,8 @@ validators==0.17.1 vine==1.3.0 wcwidth==0.2.5 websocket-client==0.57.0 +wrapt==1.14.1 xchem-db==0.1.26b0 yarl==1.5.1 zipp==3.1.0 pymysql~=0.10.1 -sphinx diff --git a/viewer/compound_set_upload.py b/viewer/compound_set_upload.py index 47b8b8eb..c01a427d 100644 --- a/viewer/compound_set_upload.py +++ b/viewer/compound_set_upload.py @@ -5,28 +5,18 @@ django.setup() from django.conf import settings -from django.core.files.storage import default_storage -from django.core.files.base import ContentFile - from rdkit import Chem from viewer.models import ( - ComputedMolecule, ScoreDescription, - NumericalScoreValues, - TextScoreValues, Protein, Target, - Molecule, ComputedSetSubmitter) -import ast import os.path -import os - -import psutil - def get_inspiration_frags(cpd, compound_set): - pass + # Don't need... + del cpd + del compound_set def process_pdb(pdb_code, target, zfile, zfile_hashvals): @@ -115,4 +105,3 @@ def get_additional_mols(filename, compound_set): return f"Missing score descriptions for: {', '.join(missing)}, please re-upload" return mols - diff --git a/viewer/cset_upload.py b/viewer/cset_upload.py index 004f8e31..8cf676be 100644 --- a/viewer/cset_upload.py +++ b/viewer/cset_upload.py @@ -248,6 +248,9 @@ def set_props(self, cpd, props, compound_set): return set_obj def set_mol(self, mol, target, compound_set, filename, zfile=None, zfile_hashvals=None): + # Don't need... + del filename + # zfile = {'zip_obj': zf, 'zf_list': zip_names} print(f'mol: {mol}') smiles = Chem.MolToSmiles(mol) @@ -290,7 +293,7 @@ def set_mol(self, mol, target, compound_set, filename, zfile=None, zfile_hashval insp_frags.append(ref) - orig = mol.GetProp('original SMILES') + _ = mol.GetProp('original SMILES') # Try to get the protein object. # This may fail. @@ -370,10 +373,10 @@ def set_descriptions(self, filename, compound_set): for key in list(description_dict.keys()): if key in descriptions_needed and key not in ['ref_mols', 'ref_pdb', 'index', 'Name', 'original SMILES']: - desc = ScoreDescription.objects.get_or_create(computed_set=compound_set, - name=key, - description=description_dict[key], - )[0] + _ = ScoreDescription.objects.get_or_create(computed_set=compound_set, + name=key, + description=description_dict[key], + ) return mols @@ -423,7 +426,7 @@ def task(self): self.process_mol(mols_to_process[i], self.target, compound_set, sdf_filename, self.zfile, self.zfile_hashvals) # check that molecules have been added to the compound set - check = ComputedMolecule.objects.filter(computed_set=compound_set) + _ = ComputedMolecule.objects.filter(computed_set=compound_set) # check compound set folder exists. cmp_set_folder = os.path.join(settings.MEDIA_ROOT, 'compound_sets') @@ -447,13 +450,28 @@ def task(self): def blank_mol_vals(sdf_file): + """Returns the submitter name, method and version (_Name) if present. + If not present the corresponding values are empty strings. + """ suppl = Chem.SDMolSupplier(sdf_file) + if not suppl: + return '', '', '' # print('%d mols detected (including blank mol)' % (len(suppl),)) blank_mol = suppl[0] + if not blank_mol: + return '', '', '' # Get submitter name/info for passing into upload to get unique name - submitter_name = blank_mol.GetProp('submitter_name') - submitter_method = blank_mol.GetProp('method') - version = blank_mol.GetProp('_Name') + submitter_name = '' + if blank_mol.HasProp('submitter_name'): + submitter_name = blank_mol.GetProp('submitter_name') + + submitter_method = '' + if blank_mol.HasProp('method'): + submitter_method = blank_mol.GetProp('method') + + version = '' + if blank_mol.HasProp('_Name'): + version = blank_mol.GetProp('_Name') return submitter_name, submitter_method, version diff --git a/viewer/download_structures.py b/viewer/download_structures.py index 7eb0d92d..c99adfea 100644 --- a/viewer/download_structures.py +++ b/viewer/download_structures.py @@ -13,7 +13,6 @@ import logging import copy import json -from pathlib import Path import pandoc from django.conf import settings @@ -110,7 +109,7 @@ def _replace_missing_sdf(molecule, code): # create the file if it doesn't exist... if not os.path.isfile(missing_path): # No file - create one. - with open(missing_path, 'w') as sd_file: + with open(missing_path, 'w', encoding='utf-8') as sd_file: # First line is the protein code, i.e. "PGN_RS02895PGA-x0346_0B" sd_file.write(f'{code}\n') # Now write the lines from the molecule sdf_info record @@ -190,7 +189,7 @@ def _read_and_patch_molecule_name(path, molecule_name=None): # We accumulate the file's content into 'content', # which we eventually return to the caller. content = '' - with open(path, 'r') as f_in: + with open(path, 'r', encoding='utf-8') as f_in: # First line (stripped) first_line = f_in.readline().strip() if first_line: @@ -259,7 +258,7 @@ def _add_file_to_sdf(combined_sdf_file, filepath): fullpath = os.path.join(media_root, filepath) if os.path.isfile(fullpath): - with open(combined_sdf_file, 'a') as f_out: + with open(combined_sdf_file, 'a', encoding='utf-8') as f_out: patched_sdf_content = _read_and_patch_molecule_name(fullpath) f_out.write(patched_sdf_content) return False @@ -321,7 +320,7 @@ def _smiles_files_zip(zip_contents, ziparchive, download_path): """Create and write the smiles file to the ZIP file """ smiles_filename = os.path.join(download_path, 'smiles.smi') - with open(smiles_filename, 'w') as smilesfile: + with open(smiles_filename, 'w', encoding='utf-8') as smilesfile: for smi in zip_contents['molecules']['smiles_info']: smilesfile.write(smi + ',') ziparchive.write( @@ -368,6 +367,8 @@ def _document_file_zip(ziparchive, download_path, original_search, host): """Create the document file This consists of a template plus an added contents description. """ + # Don't need... + del host template_file = os.path.join("/code/doc_templates", "download_readme_template.md") @@ -375,7 +376,7 @@ def _document_file_zip(ziparchive, download_path, original_search, host): readme_filepath = os.path.join(download_path, 'Readme.md') pdf_filepath = os.path.join(download_path, 'Readme.pdf') - with open(readme_filepath, "a") as readme: + with open(readme_filepath, "a", encoding="utf-8") as readme: readme.write("# Documentation for the downloaded zipfile\n") # Download links readme.write("## Download details\n") @@ -395,7 +396,7 @@ def _document_file_zip(ziparchive, download_path, original_search, host): # Download Structure from the template # (but prepare for the template file not existing)? if os.path.isfile(template_file): - with open(template_file, "r") as template: + with open(template_file, "r", encoding="utf-8") as template: readme.write(template.read()) else: logger.warning('Could not find template file (%s)', template_file) @@ -408,7 +409,7 @@ def _document_file_zip(ziparchive, download_path, original_search, host): readme.write('- '+filename+'\n') # Convert markdown to pdf file - doc = pandoc.read(open(readme_filepath, "r").read()) + doc = pandoc.read(open(readme_filepath, "r", encoding="utf-8").read()) pandoc.write(doc, file=pdf_filepath, format='latex', options=["--columns=72"]) @@ -442,7 +443,7 @@ def _create_structures_zip(target, os.makedirs(download_path, exist_ok=True) error_filename = os.path.join(download_path, "errors.csv") - error_file = open(error_filename, "w") + error_file = open(error_filename, "w", encoding="utf-8") error_file.write("Param,Code,Invalid file reference\n") errors = 0 diff --git a/viewer/migrations/0025_squonk2org_squonk2project_squonk2unit.py b/viewer/migrations/0025_squonk2org_squonk2project_squonk2unit.py new file mode 100644 index 00000000..43bb4738 --- /dev/null +++ b/viewer/migrations/0025_squonk2org_squonk2project_squonk2unit.py @@ -0,0 +1,45 @@ +# Generated by Django 3.1.14 on 2022-12-07 14:21 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('viewer', '0024_add_job_request_start_and_finish_times'), + ] + + operations = [ + migrations.CreateModel( + name='Squonk2Org', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.TextField(max_length=40)), + ('name', models.TextField(max_length=80)), + ('as_url', models.URLField()), + ('as_version', models.TextField()), + ], + ), + migrations.CreateModel( + name='Squonk2Unit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.TextField(max_length=41)), + ('name', models.TextField()), + ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='viewer.squonk2org')), + ], + ), + migrations.CreateModel( + name='Squonk2Project', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.TextField(max_length=44)), + ('name', models.TextField()), + ('product_uuid', models.TextField(max_length=44)), + ('unit', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='viewer.squonk2unit')), + ], + ), + ] diff --git a/viewer/migrations/0026_add_project_open_to_public_field.py b/viewer/migrations/0026_add_project_open_to_public_field.py new file mode 100644 index 00000000..3e0a84ec --- /dev/null +++ b/viewer/migrations/0026_add_project_open_to_public_field.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2023-01-12 11:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0025_squonk2org_squonk2project_squonk2unit'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='open_to_public', + field=models.BooleanField(default=False), + ), + ] diff --git a/viewer/migrations/0027_sessionproject_project.py b/viewer/migrations/0027_sessionproject_project.py new file mode 100644 index 00000000..56bfb07e --- /dev/null +++ b/viewer/migrations/0027_sessionproject_project.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.14 on 2023-01-24 14:23 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0026_add_project_open_to_public_field'), + ] + + operations = [ + migrations.AddField( + model_name='sessionproject', + name='project', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='viewer.project'), + ), + ] diff --git a/viewer/migrations/0028_protein_target_id_unique_constraint.py b/viewer/migrations/0028_protein_target_id_unique_constraint.py new file mode 100644 index 00000000..2595bd3a --- /dev/null +++ b/viewer/migrations/0028_protein_target_id_unique_constraint.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1.14 on 2022-10-26 09:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0027_sessionproject_project'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='protein', + unique_together={('code', 'target_id', 'prot_type')}, + ), + ] diff --git a/viewer/migrations/0029_add_job_request_project.py b/viewer/migrations/0029_add_job_request_project.py new file mode 100644 index 00000000..d4a04b22 --- /dev/null +++ b/viewer/migrations/0029_add_job_request_project.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.14 on 2023-03-07 11:46 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0028_protein_target_id_unique_constraint'), + ] + + operations = [ + migrations.AddField( + model_name='jobrequest', + name='project', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='viewer.project'), + ), + ] diff --git a/viewer/migrations/0030_jobfiletransfer_sub_path.py b/viewer/migrations/0030_jobfiletransfer_sub_path.py new file mode 100644 index 00000000..58f37c75 --- /dev/null +++ b/viewer/migrations/0030_jobfiletransfer_sub_path.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.14 on 2023-04-21 14:06 + +from django.db import migrations +import shortuuid.django_fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0029_add_job_request_project'), + ] + + operations = [ + migrations.AddField( + model_name='jobfiletransfer', + name='sub_path', + field=shortuuid.django_fields.ShortUUIDField(alphabet='abcdefghijklmnopqrstuvwxyz', length=4, max_length=4, prefix=''), + ), + ] diff --git a/viewer/migrations/0031_fix_JobFileTransfer_sub_path.py b/viewer/migrations/0031_fix_JobFileTransfer_sub_path.py new file mode 100644 index 00000000..1d7a35cb --- /dev/null +++ b/viewer/migrations/0031_fix_JobFileTransfer_sub_path.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.14 on 2023-04-24 10:15 + +from django.db import migrations +import shortuuid.django_fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0030_jobfiletransfer_sub_path'), + ] + + operations = [ + migrations.AlterField( + model_name='jobfiletransfer', + name='sub_path', + field=shortuuid.django_fields.ShortUUIDField(alphabet='abcdefghijklmnopqrstuvwxyz', length=4, max_length=4, null=True, prefix=''), + ), + ] diff --git a/viewer/models.py b/viewer/models.py index ba4dfdb7..40330f04 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -6,6 +6,8 @@ from django.core.validators import MinLengthValidator from django.conf import settings +from shortuuid.django_fields import ShortUUIDField + from simple_history.models import HistoricalRecords from viewer.target_set_config import get_mol_choices, get_prot_choices @@ -22,6 +24,8 @@ class Project(models.Model): The date the project was initiated (autofield) user_id: ManyToManyField Links to the User model + open_to_public: BooleanField + True if open to the Public """ # The title of the project_id -> userdefined title = models.CharField(max_length=200, unique=True) @@ -29,6 +33,7 @@ class Project(models.Model): init_date = models.DateTimeField(auto_now_add=True) # The users it's related to user_id = models.ManyToManyField(User) + open_to_public = models.BooleanField(default=False) class Target(models.Model): @@ -165,7 +170,7 @@ class Protein(models.Model): has_eds = models.NullBooleanField() class Meta: - unique_together = ("code", "prot_type") + unique_together = ("code", "target_id", "prot_type") class Compound(models.Model): @@ -401,7 +406,8 @@ class Meta: # Start of Session Project class SessionProject(models.Model): - """Django model for holding information about a fragalysis user project - a set of sessions saved by a user + """Django model for holding information about a fragalysis user Session Project + - a set of sessions saved by a user that belong to a Target and Project. Parameters ---------- @@ -412,17 +418,19 @@ class SessionProject(models.Model): description: Charfield A short user-defined description for the project target: ForeignKey - Foreign Key link to the relevent project target + Foreign Key link to the relevant project target + project: ForeignKey + Foreign Key link to the relevant project (optional for legacy reasons) author: ForeignKey A link to the user that created the project tags: TextField A comma separated list of user-defined tags - for searching and tagging projects - """ title = models.CharField(max_length=200) init_date = models.DateTimeField(default=timezone.now) description = models.CharField(max_length=255, default='') target = models.ForeignKey(Target, on_delete=models.CASCADE) + project = models.ForeignKey(Project, null=True, on_delete=models.CASCADE) author = models.ForeignKey(User, null=True, on_delete=models.CASCADE) tags = models.TextField(default='[]') @@ -1050,14 +1058,14 @@ class JobFileTransfer(models.Model): A foreign key link to the relevant target the file transfer is part of (required) squonk_project: CharField The name of a project that has been created in Squonk that the files will be transferred to - projects: JSONField + proteins: JSONField List of proteins to be transferred compounds: JSONField - List of coumpounds to be transferred (not used yet) + List of compounds to be transferred (not used yet) transfer_spec: JSONField Identifies for each type (protein or compound), which file types were transferred over. transfer_task_id: CharField - Task id of transfer celery task Note that if a resynchronisation is + Task id of transfer celery task Note that if a re-synchronisation is required this will be re-used. transfer_status: CharField Identifies the status of the transfer. @@ -1085,6 +1093,7 @@ class JobFileTransfer(models.Model): snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE) target = models.ForeignKey(Target, null=True, on_delete=models.CASCADE, db_index=True) squonk_project = models.CharField(max_length=200, null=True) + sub_path = ShortUUIDField(length=4, alphabet="abcdefghijklmnopqrstuvwxyz", null=True) proteins = models.JSONField(encoder=DjangoJSONEncoder, null=True) # Not used in phase 1 compounds = models.JSONField(encoder=DjangoJSONEncoder, null=True) @@ -1115,6 +1124,8 @@ class JobRequest(models.Model): A foreign key link to the relevant snapshot the file transfer is part of (required) target: ForeignKey A foreign key link to the relevant target the file transfer is part of (required) + project: ForeignKey + The Fragalysis Project record ths JobRequest is associated with squonk_project: CharField The name of a project that has been created in Squonk that the files will be transferred to squonk_job_spec: JSONField @@ -1135,9 +1146,13 @@ class JobRequest(models.Model): squonk_job_info: JSONField Squonk job information returned from the initial Squonk POST instance API call squonk_url_ext: CharField - Squonk URL information to be added to the Host URL to link to a Squonk Job + Squonk URL information to be added to the Host URL to link to a Squonk Job. + This field is populated during a call to the JobCallbackView. code: UUIDField A UUID generated by Fragalysis and passed to Squonk as part of a callback URL. + This value is used to uniquely identify the HJob in Squonk and is passed back + by squonk to provide context in calls to the JobCallBackView. + context in subs upload_task_id: CharField Celery task ID for results upload task (optional). Set when the Job completes and an automated upload follows. @@ -1174,6 +1189,7 @@ class JobRequest(models.Model): snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE) target = models.ForeignKey(Target, null=True, on_delete=models.CASCADE, db_index=True) + project = models.ForeignKey(Project, null=True, on_delete=models.CASCADE) squonk_project = models.CharField(max_length=200, null=True) squonk_job_spec = models.JSONField(encoder=DjangoJSONEncoder, null=True) # Start and finish times for the Job @@ -1188,7 +1204,7 @@ class JobRequest(models.Model): # For us this will contain a 'task_id', 'instance_id' and 'callback_token'. # The content will be a list with index '0' that's the value of the DmApiRv # 'success' variable and, at index '1', the original response message json(). - # The Job callback token will be squonk_job_info[0]['callback_token'] + # The Job callback token will be squonk_job_info[1]['callback_token'] squonk_job_info = models.JSONField(encoder=DjangoJSONEncoder, null=True) # 'squonk_url_ext' is a Squonk UI URL to obtain information about the # running instance. It's essentially the Squonk URL with the instance ID appended. @@ -1201,6 +1217,84 @@ class JobRequest(models.Model): class Meta: db_table = 'viewer_jobrequest' -# End of Squonk Job Tables + def job_has_finished(self): + """Finished if status is SUCCESS or FAILURE (or a new state of LOST). + """ + return self.job_status in [JobRequest.SUCCESS, JobRequest.FAILURE, 'LOST'] + +class Squonk2Org(models.Model): + """Django model to store Squonk2 Organisations (UUIDs) and the Account Servers + they belong to. Managed by the Squonk2Agent class. + Parameters + ---------- + uuid: TextField (40) + A Squonk2 Account Server (AS) Organisation UUID. A fixed length string + consisting of 'org-' followed by a uuid4 value, + e.g. 'org-54260047-183b-42e8-9658-385a1e1bd236' + name: TextField (80) + The name of the Squonk2 Organisation UUID (obtained form the AS). + as_url: URLField (200) + The URL of the Squonk2 Account Server that owns the organisation. + e.g. 'https://example.com/account-server-api' + as_version: TextField + The version of the AS that was first seen to own the Organisation + """ + uuid = models.TextField(max_length=40, null=False) + name = models.TextField(max_length=80, null=False) + as_url = models.URLField(null=False) + as_version = models.TextField(null=False) + +class Squonk2Unit(models.Model): + """Django model to store Squonk2 Unit (UUIDs). Managed by the Squonk2Agent class. + + Parameters + ---------- + uuid: TextField (41) + A Squonk2 Account Server (AS) Unit UUID. A fixed length string + consisting of 'unit-' followed by a uuid4 value, + e.g. 'unit-54260047-183b-42e8-9658-385a1e1bd236' + name: TextField (80) + The name used to create the Squonk2 Unit UUID + This is not limited by the actual name length imposed by the DM + target_access: ForeignKey + A Foreign Key to the Project (Proposal) the Unit belongs to, + a record that contains the "target access string". + organisation: ForeignKey + The Organisation the Unit belongs to. + """ + uuid = models.TextField(max_length=41, null=False) + name = models.TextField(null=False) + +# target_access = models.ForeignKey(Project, null=False, on_delete=models.CASCADE) + organisation = models.ForeignKey(Squonk2Org, null=False, on_delete=models.CASCADE) + +class Squonk2Project(models.Model): + """Django model to store Squonk2 Project and Product (UUIDs). + Managed by the Squonk2Agent class. + + Parameters + ---------- + uuid: TextField (44) + A Squonk2 Data Manager (DM) Project UUID. A fixed length string + consisting of 'project-' followed by a uuid4 value, + e.g. 'project-54260047-183b-42e8-9658-385a1e1bd236' + name: TextField (80) + The name of the Squonk2 Unit UUID (obtained form the AS). + product_uuid: TextField (44) + A Squonk2 Account Server (AS) Product UUID. A fixed length string + consisting of 'product-' followed by a uuid4 value, + e.g. 'product-54260047-183b-42e8-9658-385a1e1bd236' + unit: ForeignKey + The Squonk2 Unit the Product (and Project) belongs to. + """ + uuid = models.TextField(max_length=44, null=False) + name = models.TextField(null=False) + product_uuid = models.TextField(max_length=44, null=False) + + unit = models.ForeignKey(Squonk2Unit, null=False, on_delete=models.CASCADE) +# user = models.ForeignKey(User, null=False, on_delete=models.CASCADE) +# session_project = models.ForeignKey(SessionProject, null=False, on_delete=models.CASCADE) + +# End of Squonk Job Tables diff --git a/viewer/sdf_check.py b/viewer/sdf_check.py index b7b56e77..2ea5c58d 100644 --- a/viewer/sdf_check.py +++ b/viewer/sdf_check.py @@ -8,7 +8,6 @@ from rdkit import Chem import validators -import numpy as np from viewer.models import Protein, ComputedSet import datetime @@ -19,7 +18,22 @@ def check_property_descriptions(): pass def check_compound_set(description_mol, validate_dict, update=None): - y_m_d = description_mol.GetProp('generation_date').split('-') + # Must have a 'generation_date' + if not description_mol.HasProp('generation_date'): + validate_dict = add_warning(molecule_name='File error', + field='compound set', + warning_string="Molecule has no generation_date", + validate_dict=validate_dict) + return validate_dict + # That's of the form "--"... + g_date = description_mol.GetProp('generation_date') + y_m_d = g_date.split('-') + if len(y_m_d) != 3: + validate_dict = add_warning(molecule_name='File error', + field='compound set', + warning_string=f"Molecule has no generation_date is not Y-M-D (g_date)", + validate_dict=validate_dict) + return validate_dict submitter_dict = {'submitter__name': description_mol.GetProp('submitter_name'), 'submitter__email': description_mol.GetProp('submitter_email'), @@ -288,4 +302,3 @@ def check_mol_props(mol, validate_dict): validate_dict = missing_field_check(mol, field, validate_dict) return validate_dict - diff --git a/viewer/serializers.py b/viewer/serializers.py index 3f8d732e..e18eac92 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -3,6 +3,7 @@ from frag.network.query import get_full_graph from urllib.parse import urljoin +from api.security import ISpyBSafeQuerySet from api.utils import draw_mol from viewer.models import ( @@ -38,6 +39,9 @@ from rest_framework import serializers +_ISPYB_SAFE_QUERY_SET = ISpyBSafeQuerySet() + + class FileSerializer(serializers.ModelSerializer): class Meta: model = File @@ -285,12 +289,43 @@ class Meta: class ProjectSerializer(serializers.ModelSerializer): + # Field name translation (prior to refactoring the Model) + # 'tas' is the new name for 'title' + target_access_string = serializers.SerializerMethodField() + # 'authority' is the (as yet to be implemented) origin of the TAS + # For now this is fixed at "DIAMOND-ISPYB" + authority = serializers.SerializerMethodField() + # 'can_use_squonk' defines whether a user cna use Squonk for the Projetc + user_can_use_squonk = serializers.SerializerMethodField() + + def get_target_access_string(self, instance): + return instance.title + + def get_user_can_use_squonk(self, instance): + # User can use Squonk if there is a user object (i.e. they're authenticated) + # and ISPyB has the user in the Project + user = self.context['request'].user + if not user or instance.title not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(user): + return False + return True + + def get_authority(self, instance): + # Don't actually need the instance here. + # We return a hard-coded string. + del instance + return "DIAMOND-ISPYB" + class Meta: model = Project - fields = ("id", "title") + fields = ("id", + "target_access_string", + "init_date", + "authority", + "open_to_public", + "user_can_use_squonk") -class MolImageSerialzier(serializers.ModelSerializer): +class MolImageSerializer(serializers.ModelSerializer): mol_image = serializers.SerializerMethodField() @@ -311,7 +346,7 @@ class Meta: fields = ("id", "mol_image") -class CmpdImageSerialzier(serializers.ModelSerializer): +class CmpdImageSerializer(serializers.ModelSerializer): cmpd_image = serializers.SerializerMethodField() @@ -323,13 +358,13 @@ class Meta: fields = ("id", "cmpd_image") -class ProtMapInfoSerialzer(serializers.ModelSerializer): +class ProtMapInfoSerializer(serializers.ModelSerializer): map_data = serializers.SerializerMethodField() def get_map_data(self, obj): if obj.map_info: - return open(obj.map_info.path).read() + return open(obj.map_info.path, encoding='utf-8').read() else: return None @@ -338,12 +373,12 @@ class Meta: fields = ("id", "map_data", "prot_type") -class ProtPDBInfoSerialzer(serializers.ModelSerializer): +class ProtPDBInfoSerializer(serializers.ModelSerializer): pdb_data = serializers.SerializerMethodField() def get_pdb_data(self, obj): - return open(obj.pdb_info.path).read() + return open(obj.pdb_info.path, encoding='utf-8').read() class Meta: @@ -351,13 +386,13 @@ class Meta: fields = ("id", "pdb_data", "prot_type") -class ProtPDBBoundInfoSerialzer(serializers.ModelSerializer): +class ProtPDBBoundInfoSerializer(serializers.ModelSerializer): bound_pdb_data = serializers.SerializerMethodField() def get_bound_pdb_data(self, obj): if obj.bound_info: - return open(obj.bound_info.path).read() + return open(obj.bound_info.path, encoding='utf-8').read() else: return None @@ -417,7 +452,8 @@ class Meta: # GET class SessionProjectReadSerializer(serializers.ModelSerializer): target = TargetSerializer(read_only=True) - author = UserSerializer(read_only=True) + project = ProjectSerializer(read_only=True) + target = TargetSerializer(read_only=True) # This is for the new tags functionality. The old tags field is left here for backwards # compatibility session_project_tags = serializers.SerializerMethodField() @@ -429,7 +465,7 @@ def get_session_project_tags(self, obj): class Meta: model = SessionProject fields = ('id', 'title', 'init_date', 'description', - 'target', 'author', 'tags', 'session_project_tags') + 'target', 'project', 'author', 'tags', 'session_project_tags') # (POST, PUT, PATCH) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py new file mode 100644 index 00000000..0d25a8df --- /dev/null +++ b/viewer/squonk2_agent.py @@ -0,0 +1,1021 @@ +"""An agent for the Squonk2 (Data Manger and Account Server) API. +This module 'simplifies' the use of the Squonk2 Python client package. +""" + +# Refer to the accompanying low-level-design document: - +# https://docs.google.com/document/d/1lFpN29dK1luz80lwSGi0Rnj1Rqula2_CTRuyWDUBu14 + +from collections import namedtuple +import logging +import os +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import ParseResult, urlparse +from urllib3.exceptions import InsecureRequestWarning +from urllib3 import disable_warnings + +from squonk2.auth import Auth +from squonk2.as_api import AsApi, AsApiRv +from squonk2.dm_api import DmApi, DmApiRv +import requests +from requests import Response +from wrapt import synchronized + +from api.security import ISpyBSafeQuerySet +from viewer.models import User, Project, Target +from viewer.models import Squonk2Project, Squonk2Org, Squonk2Unit + +_LOGGER: logging.Logger = logging.getLogger(__name__) + +# Response value for the agent methods. +# It's a boolean and an optional string (used for errors or response content) +Squonk2AgentRv: namedtuple = namedtuple('Squonk2AgentRv', ['success', 'msg']) +SuccessRv: Squonk2AgentRv = Squonk2AgentRv(success=True, msg=None) + +# Parameters common to each named Tuple +CommonParams: namedtuple = namedtuple("CommonParams", + ["user_id", + "access_id", + "target_id", + "session_id"]) + +# Named tuples are used to pass parameters to the agent methods. +# RunJob, used in run_job() +RunJobParams: namedtuple = namedtuple("RunJob", ["common", + "job_spec", + "callback_url"]) + +# Send, used in send() +# Access ID is the Fragalysis Project record ID +# Session ID is the Fragalysis SessionProject record ID +# Target ID is the Fragalysis Target record ID +# Snapshot ID is the Fragalysis Snapshot record ID +SendParams: namedtuple = namedtuple("Send", ["common", + "snapshot_id"]) + +# Parameters for the grant_access() method. +AccessParams: namedtuple = namedtuple("Access", ["username", + "project_uuid"]) + +_SUPPORTED_PRODUCT_FLAVOURS: List[str] = ["BRONZE", "SILVER", "GOLD"] + +# Squonk2 Have defined limits - assumed here. +# verify with your Squonk2 installation. +# How long are Squonk2 'names'? +_SQ2_MAX_NAME_LENGTH: int = 80 + +# A slug used for names this Fragalysis will create +# and a prefix string. So Squonk2 objects will be called "Fragalysis {slug}" +_MAX_SLUG_LENGTH: int = 10 +_SQ2_NAME_PREFIX: str = "Fragalysis" + +# Built-in +_SQ2_PRODUCT_TYPE: str = 'DATA_MANAGER_PROJECT_TIER_SUBSCRIPTION' + +# True if the code's in Test Mode +_TEST_MODE: bool = False + + +class Squonk2Agent: + """Helper class that simplifies access to the Squonk2 Python client. + Users shouldn't instantiate the class directly, instead they should + get access to the class singleton via a call to 'get_squonk2_agent()'. + + The class methods protect the caller from using them unless a) the class has + sufficient configuration and b) the Squonk2 services are 'alive'. + """ + + def __init__(self): + """Initialise the instance, loading from the environment. + """ + + # Primary configuration of the module is via the container environment. + # We need to recognise that some or all of these may not be defined. + # All run-time config that's required is given a __CFG prefix to + # simplify checking whether all that's required has been defined. + # + # The SQUONK2_SLUG is limited to 10 characters, when combined with + # "Fragalysis {SLUG} ", this leaves (80-22) 58 characters for the + # use with the target-access-string and session project strings + # to form Squonk2 Unit and Project names. + self.__CFG_SQUONK2_ASAPI_URL: Optional[str] =\ + os.environ.get('SQUONK2_ASAPI_URL') + self.__CFG_SQUONK2_DMAPI_URL: Optional[str] =\ + os.environ.get('SQUONK2_DMAPI_URL') + self.__CFG_SQUONK2_UI_URL: Optional[str] =\ + os.environ.get('SQUONK2_UI_URL') + self.__CFG_SQUONK2_ORG_UUID: Optional[str] =\ + os.environ.get('SQUONK2_ORG_UUID') + self.__CFG_SQUONK2_UNIT_BILLING_DAY: Optional[str] =\ + os.environ.get('SQUONK2_UNIT_BILLING_DAY') + self.__CFG_SQUONK2_PRODUCT_FLAVOUR: Optional[str] =\ + os.environ.get('SQUONK2_PRODUCT_FLAVOUR') + self.__CFG_SQUONK2_SLUG: Optional[str] =\ + os.environ.get('SQUONK2_SLUG', '')[:_MAX_SLUG_LENGTH] + self.__CFG_SQUONK2_ORG_OWNER: Optional[str] =\ + os.environ.get('SQUONK2_ORG_OWNER') + self.__CFG_SQUONK2_ORG_OWNER_PASSWORD: Optional[str] =\ + os.environ.get('SQUONK2_ORG_OWNER_PASSWORD') + self.__CFG_OIDC_AS_CLIENT_ID: Optional[str] = \ + os.environ.get('OIDC_AS_CLIENT_ID') + self.__CFG_OIDC_DM_CLIENT_ID: Optional[str] = \ + os.environ.get('OIDC_DM_CLIENT_ID') + self.__CFG_OIDC_KEYCLOAK_REALM: Optional[str] = \ + os.environ.get('OIDC_KEYCLOAK_REALM') + + # Optional config (no '__CFG_' prefix) + self.__DUMMY_TARGET_TITLE: Optional[str] =\ + os.environ.get('DUMMY_TARGET_TITLE') + self.__DUMMY_USER: Optional[str] =\ + os.environ.get('DUMMY_USER') + self.__DUMMY_TAS: Optional[str] =\ + os.environ.get('DUMMY_TAS') + self.__SQUONK2_VERIFY_CERTIFICATES: Optional[str] = \ + os.environ.get('SQUONK2_VERIFY_CERTIFICATES') + + # The integer billing day, valid if greater than zero + self.__unit_billing_day: int = 0 + # True if configured... + self.__configuration_checked: bool = False + self.__configured: bool = False + # Ignore cert errors? (no) + self.__verify_certificates: bool = True + + # The record ID of the Squonk2Org for this deployment. + # Set on successful 'pre-flight-check' + self.__org_record: Optional[Squonk2Org] = None + + self.__org_owner_as_token: str = '' + self.__org_owner_dm_token: str = '' + self.__keycloak_hostname: str = '' + self.__keycloak_realm: str = '' + + # The Safe QuerySet from the security module. + # Used when we are given a tas (target access string). + # It allows us to check that a user is permitted to use the access ID + # and relies on ISPyB credentials present in the environment. + self.__ispyb_safe_query_set: ISpyBSafeQuerySet = ISpyBSafeQuerySet() + + def _get_user_name(self, user_id: int) -> str: + # Gets the username (if id looks sensible) + # or a fixed value (used for testing) + if user_id == 0: + assert _TEST_MODE + _LOGGER.warning('Caution - in TEST mode, using __DUMMY_USER (%s)', + self.__DUMMY_USER) + + user_name: str = self.__DUMMY_USER + if user_id: + user: User = User.objects.filter(id=user_id).first() + assert user + user_name = user.username + assert user_name + return user_name + + def _get_target_access_string(self, access_id: int) -> str: + # Gets the Target Access String (if id looks sensible) + # or a fixed value (used for testing) + if access_id == 0: + assert _TEST_MODE + _LOGGER.warning('Caution - in TEST mode, using __DUMMY_TAS (%s)', + self.__DUMMY_TAS) + + # Get the "target access string" (test mode or otherwise) + target_access_string: str = self.__DUMMY_TAS + if access_id: + project: Optional[Project] = Project.objects.filter(id=access_id).first() + assert project + target_access_string = project.title + assert target_access_string + return target_access_string + + def _get_target_title(self, target_id: int) -> str: + # Gets the Target title (if it looks sensible) + # or a fixed value (used for testing) + if target_id == 0: + assert _TEST_MODE + _LOGGER.warning('Caution - in TEST mode, using __DUMMY_TARGET_TITLE (%s)', + self.__DUMMY_TARGET_TITLE) + + target_title: str = self.__DUMMY_TARGET_TITLE + if target_id: + target: Target = Target.objects.filter(id=target_id).first() + assert target + target_title = target.title + assert target_title + return target_title + + def _build_unit_name(self, target_access_string: str) -> Tuple[str, str]: + assert target_access_string + # AS Units are named using the Target Access String (TAS) + # which, in Diamond will be a "visit" string like "lb00000-1" + name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} /{target_access_string}/' + return name[:_SQ2_MAX_NAME_LENGTH], name + + def _build_product_name(self, username: str, target_title: str) -> Tuple[str, str]: + """Builds a Product name, returning the truncated and un-truncated form""" + assert username + assert target_title + # AS Products are named using the user and the session + # (there's a 1:1 mapping to DM Projects) + + # The Product name characters are not restricted + identifier: str = f'{username}::{target_title}' + name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} {identifier}' + return name[:_SQ2_MAX_NAME_LENGTH], name + + def _get_squonk2_owner_tokens(self) -> Optional[Tuple[str, str]]: + """Gets access tokens for the Squonk2 organisation owner. + This sets the __keycloak_hostname member and also returns the tokens, + getting one for the AS and one for the DM. + """ + assert self.__keycloak_hostname + + _LOGGER.debug('__keycloak_hostname="%s" __keycloak_realm="%s"' + ' dm-client=%s as-client=%s org_owner=%s', + self.__keycloak_hostname, + self.__keycloak_realm, + self.__CFG_OIDC_DM_CLIENT_ID, + self.__CFG_OIDC_AS_CLIENT_ID, + self.__CFG_OIDC_AS_CLIENT_ID) + + self.__org_owner_as_token = Auth.get_access_token( + keycloak_url="https://" + self.__keycloak_hostname + "/auth", + keycloak_realm=self.__keycloak_realm, + keycloak_client_id=self.__CFG_OIDC_AS_CLIENT_ID, + username=self.__CFG_SQUONK2_ORG_OWNER, + password=self.__CFG_SQUONK2_ORG_OWNER_PASSWORD, + ) + if not self.__org_owner_as_token: + _LOGGER.warning('Failed to get access token for AS Organisation owner') + return None + + self.__org_owner_dm_token = Auth.get_access_token( + keycloak_url="https://" + self.__keycloak_hostname + "/auth", + keycloak_realm=self.__keycloak_realm, + keycloak_client_id=self.__CFG_OIDC_DM_CLIENT_ID, + username=self.__CFG_SQUONK2_ORG_OWNER, + password=self.__CFG_SQUONK2_ORG_OWNER_PASSWORD, + ) + if not self.__org_owner_dm_token: + _LOGGER.warning('Failed to get access token for DM as AS Organisation owner') + return None + + # OK if we get here + return self.__org_owner_as_token, self.__org_owner_dm_token + + def _pre_flight_checks(self) -> Squonk2AgentRv: + """Execute pre-flight checks, + can be called multiple times, it acts only once. + """ + + # If a Squonk2Org record exists its UUID cannot have changed. + # We cannot change the organisation once deployed. The corresponding Units, + # Products and Projects are organisation-specific. The Squonk2Org table + # records the organisation ID and the Account Server URL where the ID + # is valid. None of these values can change once deployed. + + squonk2_org: Optional[Squonk2Org] = Squonk2Org.objects.all().first() + if squonk2_org and squonk2_org.uuid != self.__CFG_SQUONK2_ORG_UUID: + msg: str = f'Configured Squonk2 Organisation ({self.__CFG_SQUONK2_ORG_UUID})'\ + f' does not match pre-existing record ({squonk2_org.uuid})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # OK, so the ORG UUID has not changed. + # Is it known to the configured AS? + if not self._get_squonk2_owner_tokens(): + msg = 'Failed to get AS or DM token for organisation owner' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Get the ORG from the AS API. + # If it knows the org the response will be successful, + # and we'll also have the Org's name. + as_o_rv = AsApi.get_organisation(self.__org_owner_as_token, + org_id=self.__CFG_SQUONK2_ORG_UUID) + if not as_o_rv.success: + msg = 'Failed to get AS Organisation' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # The org is known to the AS. + # Get the AS API version (for reference) + as_v_rv: AsApiRv = AsApi.get_version() + if not as_v_rv.success: + msg = 'Failed to get version from AS' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + as_version: str = as_v_rv.msg['version'] + + # If there's no Squonk2Org record, create one, + # recording the ORG ID and the AS we used to verify it exists. + if not squonk2_org: + _LOGGER.info('Creating NEW Squonk2Org record for %s.' + ' as-url=%s as-org="%s" as-version=%s', + self.__CFG_SQUONK2_ORG_UUID, + self.__CFG_SQUONK2_ASAPI_URL, + as_o_rv.msg['name'], + as_version) + squonk2_org = Squonk2Org(uuid=self.__CFG_SQUONK2_ORG_UUID, + name=as_o_rv.msg['name'], + as_url=self.__CFG_SQUONK2_ASAPI_URL, + as_version=as_version) + squonk2_org.save() + _LOGGER.info('Created Squonk2Org record for %s', + self.__CFG_SQUONK2_ORG_UUID) + else: + _LOGGER.debug('Squonk2Org for %s "%s" already exists - nothing to do', + squonk2_org.uuid, + squonk2_org.name) + + # Keep the record ID for future use. + self.__org_record = squonk2_org + + # Organisation is known to AS, and it hasn't changed. + return SuccessRv + + def _get_or_create_unit(self, + unit_name_truncated: str, + unit_name_full: str) -> Squonk2AgentRv: + """Gets an exiting Unit or creates a new one + returning its UUID as the msg content. + """ + # Get existing Units for our Organisation + org_uuid: str = self.__org_record.uuid + as_rv: AsApiRv = AsApi.get_units(self.__org_owner_as_token, org_id=org_uuid) + if not as_rv.success: + msg: str = as_rv.msg['error'] + _LOGGER.error('Failed to get Units for Organisation "%s"', org_uuid) + return Squonk2AgentRv(success=False, msg=msg) + # Iterate through them, looking for ours... + _LOGGER.info('Got %s Units for Organisation "%s"', len(as_rv.msg['units']), org_uuid) + for unit in as_rv.msg['units']: + if unit['name'] == unit_name_truncated: + unit_uuid: str = unit['id'] + _LOGGER.info('...and one was ours (%s)', unit_uuid) + return Squonk2AgentRv(success=True, msg=unit_uuid) + + # Not found, create it... + _LOGGER.info('No exiting Unit, creating NEW Unit "%s" (for "%s")', + unit_name_full, + unit_name_truncated) + as_rv = AsApi.create_unit(self.__org_owner_as_token, + unit_name=unit_name_truncated, + org_id=org_uuid, + billing_day=self.__unit_billing_day) + if not as_rv.success: + msg = as_rv.msg['error'] + _LOGGER.error('Failed to create Unit "%s" (for "%s")', + unit_name_full, unit_name_truncated) + return Squonk2AgentRv(success=False, msg=msg) + + unit_uuid = as_rv.msg['id'] + _LOGGER.info('Created NEW Unit "%s"', unit_uuid) + return Squonk2AgentRv(success=True, msg=unit_uuid) + + def _get_or_create_product(self, name_truncated: str, unit_uuid: str) -> Squonk2AgentRv: + + # Get existing Products for the Unit + as_rv: AsApiRv = AsApi.get_products_for_unit(self.__org_owner_as_token, + unit_id=unit_uuid) + if not as_rv.success: + msg: str = as_rv.msg['error'] + _LOGGER.error('Failed to get Products for Unit "%s"', unit_uuid) + return Squonk2AgentRv(success=False, msg=msg) + + # Iterate through them, looking for ours... + for product in as_rv.msg['products']: + if product['product']['name'] == name_truncated: + product_uuid: str = product['product']['id'] + _LOGGER.info('Found pre-existing AS Product "%s"', product_uuid) + return Squonk2AgentRv(success=True, msg=product_uuid) + + # No existing Product with this name... + _LOGGER.info('No existing Product, creating NEW AS Product "%s" (for "%s")', + name_truncated, + unit_uuid) + as_rv = AsApi.create_product(self.__org_owner_as_token, + product_name=name_truncated, + unit_id=unit_uuid, + product_type=_SQ2_PRODUCT_TYPE, + flavour=self.__CFG_SQUONK2_PRODUCT_FLAVOUR) + if not as_rv.success: + msg = f'Failed to create AS Product ({as_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + product_uuid = as_rv.msg['id'] + msg = f'Created NEW AS Product {product_uuid}...' + _LOGGER.info(msg) + return Squonk2AgentRv(success=True, msg=product_uuid) + + def _get_or_create_project(self, name_truncated: str, product_uuid: str) -> Squonk2AgentRv: + """Gets existing DM Projects (that belong to the given Product) + to see if ours exists, if not a new one is created. + """ + # TODO + dm_rv: DmApiRv = DmApi.get_available_projects(self.__org_owner_as_token) + if not dm_rv.success: + msg: str = dm_rv.msg['error'] + _LOGGER.error('Failed to get Projects') + return Squonk2AgentRv(success=False, msg=msg) + + # Iterate through them, looking for ours... + for project in dm_rv.msg['projects']: + if project['name'] == name_truncated and 'product_id' in project and project['product_id'] == product_uuid: + project_uuid: str = project['project_id'] + _LOGGER.info('Found pre-existing DM Project "%s"', project_uuid) + return Squonk2AgentRv(success=True, msg=project_uuid) + + # No existing Project + _LOGGER.info('No existing Project, creating NEW DM Project "%s" (for "%s")', + name_truncated, + product_uuid) + dm_rv = DmApi.create_project(self.__org_owner_dm_token, + project_name=name_truncated, + private=True, + as_tier_product_id=product_uuid) + if not dm_rv.success: + msg = f'Failed to create DM Project ({dm_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + project_uuid: str = dm_rv.msg['project_id'] + msg = f'Created NEW DM Project {project_uuid}...' + return Squonk2AgentRv(success=True, msg=project_uuid) + + def _delete_as_product(self, product_uuid: str) -> None: + """Used in error conditions to remove a previously created Product. + If this fails there's nothing else we can do so we just return regardless. + """ + _LOGGER.warning('Deleting AS Product %s...', product_uuid) + + as_rv: AsApiRv = AsApi.delete_product(self.__org_owner_as_token, + product_id=product_uuid) + if not as_rv.success: + _LOGGER.error('Failed to delete AS Product %s', product_uuid) + return + + _LOGGER.warning('Deleted AS Product %s', product_uuid) + + def _delete_dm_project(self, project_uuid: str) -> None: + """Used in error conditions to remove a previously created Project. + If this fails there's nothing else we can do so we just return regardless. + """ + _LOGGER.warning('Deleting DM Project %s...', project_uuid) + + dm_rv: DmApiRv = DmApi.delete_project(self.__org_owner_dm_token, + project_id=project_uuid) + if not dm_rv.success: + _LOGGER.error('Failed to delete DM Project %s', project_uuid) + return + + _LOGGER.warning('Deleted DM Project %s', project_uuid) + + def _create_product_and_project(self, + unit: Squonk2Unit, + user_name: str, + session_title: str, + params: CommonParams) -> Squonk2AgentRv: + """Called if a Product (and Project) needs to be created. If successful, + this call returns a dictionary in the response "msg" that contains values + for the keys "sq2_product_uuid" and "sq2_project_uuid". + """ + assert unit + assert user_name + assert session_title + assert params + + # Create an AS Product. + name_truncated, _ = self._build_product_name(user_name, session_title) + msg: str = f'Creating AS Product "{name_truncated}" (unit={unit.uuid})...' + _LOGGER.info(msg) + + sq2_rv: Squonk2AgentRv = self._get_or_create_product(name_truncated, unit.uuid) + if not sq2_rv.success: + msg = f'Failed to create AS Product ({sq2_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + product_uuid: str = sq2_rv.msg + msg = f'Got or created AS Product {product_uuid}' + _LOGGER.info(msg) + + # Create a DM Project (using the same name we used for the AS Product) + msg = f'Continuing by creating DM Project "{name_truncated}"...' + _LOGGER.info(msg) + + sq2_rv: Squonk2AgentRv = self._get_or_create_project(name_truncated, product_uuid) + if not sq2_rv.success: + msg = f'Failed to create DM Project ({sq2_rv.msg})' + _LOGGER.error(msg) + # First delete the AS Product it should have been attached to + self._delete_as_product(product_uuid) + # Then leave... + return Squonk2AgentRv(success=False, msg=msg) + project_uuid: str = sq2_rv.msg + msg = f'Got or created DM Project {project_uuid}...' + _LOGGER.info(msg) + + # Add the user as an Editor to the Project + msg = f'Adding "{user_name}" to DM Project {project_uuid} as Editor...' + _LOGGER.info(msg) + dm_rv = DmApi.add_project_editor(self.__org_owner_dm_token, + project_id=project_uuid, + editor=user_name) + if not dm_rv.success: + msg = f'Failed to add "{user_name}" to DM Project ({dm_rv.msg})' + _LOGGER.error(msg) + _LOGGER.warning('Rolling back DM Project and AS Product creation...') + # First delete the DM Project amd the corresponding AS Product... + self._delete_dm_project(project_uuid) + self._delete_as_product(product_uuid) + # Then leave... + return Squonk2AgentRv(success=False, msg=msg) + + msg = f'"{user_name}" is now an Editor of DM Project {project_uuid}' + _LOGGER.info(msg) + + # If the second call fails - delete the object created in the first + + response_msg: Dict[str, Any] = {"sq2_project_uuid": project_uuid, + "sq2_product_uuid": product_uuid} + return Squonk2AgentRv(success=True, msg=response_msg) + + def _ensure_unit(self, target_access_string: int) -> Squonk2AgentRv: + """Gets or creates a Squonk2 Unit based on a customer's "target access string" + (TAS). If a Unit is created its name will begin with the text "Fragalysis " + followed by the configured 'SQUONK2_SLUG' (chosen to be unique between all + Fragalysis instances that share the same Squonk2 service) and then the + TAS. In DLS the TAS is essentially the "proposal". + + On success the returned message is used to carry the Squonk2 project UUID. + """ + if not self.__org_record: + msg: str = 'The Squonk2Org record does not match' \ + ' the configured SQUONK2_ORG_UUID.' \ + ' You cannot change the SQUONK2_ORG_UUID once it has been used' + return Squonk2AgentRv(success=False, msg=msg) + + # Now we check and create a Squonk2Unit... + unit_name_truncated, unit_name_full = self._build_unit_name(target_access_string) + sq2_unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(name=unit_name_full).first() + if not sq2_unit: + _LOGGER.info('No existing Squonk2Unit for "%s"', target_access_string) + # Get the list of Units from Squonk. + sq2a_rv: Squonk2AgentRv = self._get_or_create_unit(unit_name_truncated, unit_name_full) + if not sq2a_rv.success: + _LOGGER.error('Failed to create Unit "%s" (%s)', target_access_string, sq2a_rv.msg) + return Squonk2AgentRv(success=False, msg=sq2a_rv.msg) + + unit_uuid: str = sq2a_rv.msg + sq2_unit = Squonk2Unit(uuid=unit_uuid, + name=unit_name_full, + organisation_id=self.__org_record.id) + sq2_unit.save() + _LOGGER.info('Created Squonk2Unit %s "%s" (for "%s")', + unit_uuid, + unit_name_full, + target_access_string) + else: + _LOGGER.debug('Squonk2Unit %s "%s" already exists (for "%s") - nothing to do', + sq2_unit.uuid, + unit_name_full, + target_access_string) + + return Squonk2AgentRv(success=True, msg=sq2_unit) + + def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: + """Gets or creates a Squonk2 Project, used as the destination of files + and job executions. Each Project requires an AS Product + (tied to the User and Session) and Unit (tied to the Proposal/Project). + + The proposal is expected to be valid for a given user, this method does not + check whether the user/proposal combination - it assumes that what's been + given has been checked. + + On success the returned message is used to carry the Squonk2Project record. + + For testing the target and user IDs are permitted to be 0. + """ + assert c_params + assert isinstance(c_params, CommonParams) + + target_access_string = self._get_target_access_string(c_params.access_id) + assert target_access_string + + # A Squonk2Unit must exist for the Target Access String. + rv: Squonk2AgentRv = self._ensure_unit(target_access_string) + if not rv.success: + return rv + unit: Squonk2Unit = rv.msg + + user_name: str = self._get_user_name(c_params.user_id) + target_title: str = self._get_target_title(c_params.target_id) + assert user_name + assert target_title + + _, name_full = self._build_product_name(user_name, target_title) + sq2_project: Optional[Squonk2Project] = Squonk2Project.objects.filter(name=name_full).first() + if not sq2_project: + msg = f'No existing Squonk2Project for "{name_full}"' + _LOGGER.info(msg) + # Need to call upon Squonk2 to create a 'Product' + # (and corresponding 'Product'). + rv = self._create_product_and_project(unit, user_name, target_title, c_params) + if not rv.success: + msg = f'Failed creating AS Product or DM Project ({rv.msg})' + _LOGGER.error(msg) + return rv + + # Now record these new remote objects in a new + # Squonk2Project record. As it's worked we're given + # a dictionary with keys "sq2_project_uuid" and "sq2_product_uuid" + sq2_project = Squonk2Project(uuid=rv.msg['sq2_project_uuid'], + name=name_full, + product_uuid=rv.msg['sq2_product_uuid'], + unit_id=unit.id) + sq2_project.save() + msg = f'Created NEW Squonk2Project for {sq2_project.uuid} "{name_full}"' + _LOGGER.info(msg) + else: + msg = f'Squonk2Project for {sq2_project.uuid} "{name_full}" already exists - nothing to do' + _LOGGER.debug(msg) + + return Squonk2AgentRv(success=True, msg=sq2_project) + + def _verify_access(self, c_params: CommonParams) -> Squonk2AgentRv: + """Checks the user has access to the project. + """ + access_id: int = c_params.access_id + assert access_id + project: Optional[Project] = Project.objects.filter(id=access_id).first() + if not project: + msg = f'Access ID (Project) {access_id} does not exist' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Ensure that the user is allowed to use the given access ID. + # Even on public projects the user must be part of the project + # to use Squonk. + user: User = User.objects.filter(id=c_params.user_id).first() + assert user + target_access_string = self._get_target_access_string(access_id) + assert target_access_string + proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) + if not target_access_string in proposal_list: + msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ + f' (access_id={access_id}). Only {proposal_list})' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return SuccessRv + + @synchronized + def get_ui_url(self): + """Returns the UI URL, if configured. + """ + return self.__CFG_SQUONK2_UI_URL + + @synchronized + def configured(self) -> Squonk2AgentRv: + """Returns True if the module appears to be configured, + i.e. all the environment variables appear to be set. + """ + + # To prevent repeating the checks, all of which are based on + # static (environment) variables, if we've been here before + # just return our previous result. + if self.__configuration_checked: + return Squonk2AgentRv(success=self.__configured, msg=None) + + self.__configuration_checked = True + for name, value in self.__dict__.items(): + # All required configuration has a class '__CFG' prefix + if name.startswith('_Squonk2Agent__CFG_'): + if value is None: + cfg_name: str = name.split('_Squonk2Agent__CFG_')[1] + msg = f'{cfg_name} is not set' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # If we get here all the required configuration variables are set + + # Is the slug too long? + # Limited to 10 characters + if len(self.__CFG_SQUONK2_SLUG) > _MAX_SLUG_LENGTH: + msg = f'Slug is longer than {_MAX_SLUG_LENGTH} characters'\ + f' ({self.__CFG_SQUONK2_SLUG})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Extract hostname and realm from the legacy variable + # i.e. we need 'example.com' and 'xchem' + # from 'https://example.com/auth/realms/xchem' + url: ParseResult = urlparse(self.__CFG_OIDC_KEYCLOAK_REALM) + self.__keycloak_hostname = url.hostname + self.__keycloak_realm = os.path.split(url.path)[1] + + # Can we translate the billing day to an integer? + if not self.__CFG_SQUONK2_UNIT_BILLING_DAY.isdigit(): + msg = 'SQUONK2_UNIT_BILLING_DAY is set'\ + ' but the value is not a number'\ + f' ({ self.__CFG_SQUONK2_UNIT_BILLING_DAY})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + self.__unit_billing_day = int(self.__CFG_SQUONK2_UNIT_BILLING_DAY) + if self.__unit_billing_day < 1: + msg = 'SQUONK2_UNIT_BILLING_DAY cannot be less than 1' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Product tier flavour must be one of a known value. + # It's stored in the object's self.__product_flavour as an upper-case value + if not self.__CFG_SQUONK2_PRODUCT_FLAVOUR in _SUPPORTED_PRODUCT_FLAVOURS: + msg = f'SQUONK2_PRODUCT_FLAVOUR ({self.__CFG_SQUONK2_PRODUCT_FLAVOUR})' \ + ' is not supported' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Don't verify Squonk2 SSL certificates? + if self.__SQUONK2_VERIFY_CERTIFICATES and self.__SQUONK2_VERIFY_CERTIFICATES.lower() == 'no': + self.__verify_certificates = False + disable_warnings(InsecureRequestWarning) + + # OK - it all looks good. + # Mark as 'configured' + self.__configured = True + + return SuccessRv + + @synchronized + def ping(self) -> Squonk2AgentRv: + """Returns True if all the Squonk2 installations + referred to by the URLs respond. + + We also validate that the organisation supplied is known to the Account Server + by calling on '_pre_flight_checks()'. If the org is known a Squonk2Org record + is created, if not the ping fails. + """ + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + if not self.configured(): + msg = 'Not configured' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Check the UI, DM and AS... + + resp: Optional[Response] = None + url: str = self.__CFG_SQUONK2_UI_URL + try: + resp: Response = requests.head(url, verify=self.__verify_certificates) + except: + _LOGGER.error('Exception checking UI at %s', url) + if resp is None or resp.status_code != 200: + msg = f'Squonk2 UI is not responding from {url}' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + resp = None + url = f'{self.__CFG_SQUONK2_DMAPI_URL}/api' + try: + resp = requests.head(url, verify=self.__verify_certificates) + except Exception: # pylint: disable=broad-except + _LOGGER.error('Exception checking DM at %s', url) + if resp is None or resp.status_code != 308: + msg = f'Squonk2 DM is not responding from {url}' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + resp = None + url = f'{self.__CFG_SQUONK2_ASAPI_URL}/api' + try: + resp = requests.head(url, verify=self.__verify_certificates) + except: + _LOGGER.error('Exception checking AS at %s', url) + if resp is None or resp.status_code != 308: + msg = f'Squonk2 AS is not responding from {url}' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # OK so far. + # Is the configured organisation known to the AS (and has it changed?) + status, msg = self._pre_flight_checks() + if not status: + msg = f'Failed pre-flight checks ({msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Everything's responding if we get here... + return SuccessRv + + @synchronized + def can_run_job(self, rj_params: RunJobParams) -> Squonk2AgentRv: + """Executes a Job on a Squonk2 installation. + """ + assert rj_params + assert isinstance(rj_params, RunJobParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Protect against lack of config or connection/setup issues... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return self._verify_access(c_params=rj_params.common) + + @synchronized + def can_send(self, s_params: SendParams) -> Squonk2AgentRv: + """A blocking method that checks whether a user can send files to Squonk2. + """ + assert s_params + assert isinstance(s_params, SendParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API**MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return self._verify_access(c_params=s_params.common) + + @synchronized + def send(self, s_params: SendParams) -> Squonk2AgentRv: + """A blocking method that takes care of sending a set of files to + the configured Squonk2 installation. + """ + assert s_params + assert isinstance(s_params, SendParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API**MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + rv_access: Squonk2AgentRv = self._verify_access(s_params.common) + if not rv_access.success: + return rv_access + + rv_u: Squonk2AgentRv = self._ensure_project(s_params.common) + if not rv_u.success: + msg = f'Failed to create corresponding Squonk2 Project (msg={rv_u.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return SuccessRv + + @synchronized + def ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: + """A blocking method that takes care of the provisioning of the + required Squonk2 environment. For Fragalysis this entails the + creation of a 'Squonk2 Project' (which also requires a 'Unit' and 'Product'). + + If successful the Corresponding Squonk2Project record is returned as + the response 'msg' value. + """ + assert c_params + assert isinstance(c_params, CommonParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API **MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + rv_access: Squonk2AgentRv = self._verify_access(c_params) + if not rv_access.success: + return rv_access + + rv_u: Squonk2AgentRv = self._ensure_project(c_params) + if not rv_u.success: + msg = f'Failed to create corresponding Squonk2 Project (msg={rv_u.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return rv_u + + @synchronized + def grant_access(self, a_params: AccessParams) -> Squonk2AgentRv: + """A blocking method that takes care of sending a set of files to + the configured Squonk2 installation. + """ + assert a_params + assert isinstance(a_params, AccessParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API **MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + dm_rv: DmApiRv = DmApi.add_project_observer(self.__org_owner_dm_token, + project_id=a_params.project_uuid, + observer=a_params.username) + if not dm_rv.success: + msg = f'Failed to add DM Project Observer ({dm_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return SuccessRv + + @synchronized + def get_instance_execution_status(self, callback_context: str) -> Squonk2AgentRv: + """A blocking method that attempt to get the execution status (success/failure) + of an instance (a Job) based on the given callback context. The status (string) + is returned as the Squonk2AgentRv.msg value. + """ + assert callback_context + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API **MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # To do this we actually get the DM tasks associated with the + # callback context that Fragalysis provided. + # For the filters we provide we should only get one Task. + dm_rv: DmApiRv = DmApi.get_tasks(self.__org_owner_dm_token, + exclude_removal=True, + exclude_purpose='FILE.DATASET.PROJECT', + instance_callback_context=callback_context) + if not dm_rv.success: + msg = f'Failed to get DM Tasks ({dm_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Find the task that relates to our instance. + # We return 'LOST' if a task cannot be found, + # otherwise it's one of None, SUCCESS or FAILURE + i_status: Optional[str] = 'LOST' + num_tasks: int = len(dm_rv.msg['tasks']) + if num_tasks == 1: + i_task = dm_rv.msg['tasks'][0] + if i_task['done']: + i_status = 'FAILURE' if i_task['exit_code'] != 0 else 'SUCCESS' + else: + i_status = None + else: + msg = f'More than one Task found ({num_tasks}) for callback context "{callback_context}"' + _LOGGER.warning(msg) + + if i_status and i_status == 'LOST': + msg = f'No Task found for callback context "{callback_context}", assume "LOST"' + _LOGGER.warning(msg) + + return Squonk2AgentRv(success=True, msg=i_status) + +# A placeholder for the Agent object +_AGENT_SINGLETON: Optional[Squonk2Agent] = None + +def get_squonk2_agent() -> Squonk2Agent: + """Returns a 'singleton'. + """ + global _AGENT_SINGLETON # pylint: disable=global-statement + + if _AGENT_SINGLETON: + return _AGENT_SINGLETON + _LOGGER.debug("Creating new Squonk2Agent...") + _AGENT_SINGLETON = Squonk2Agent() + _LOGGER.debug("Created") + + return _AGENT_SINGLETON diff --git a/viewer/squonk_job_file_transfer.py b/viewer/squonk_job_file_transfer.py index 1bddf647..26a85301 100644 --- a/viewer/squonk_job_file_transfer.py +++ b/viewer/squonk_job_file_transfer.py @@ -7,7 +7,6 @@ from django.conf import settings from rest_framework import status from squonk2.dm_api import DmApi -from rdkit import Chem from celery.utils.log import get_task_logger from viewer.utils import ( @@ -249,9 +248,13 @@ def process_file_transfer(auth_token, trans_dir = create_media_sub_directory(trans_sub_dir) # location in squonk project where files will reside - # e.g. "/fragalysis-files/Mpro" + # e.g. "/fragalysis-files/hjyx/Mpro" for new transfers, + # "/fragalysis-files/Mpro" for existing transfers target = job_transfer.target - squonk_directory = '/' + settings.SQUONK2_MEDIA_DIRECTORY + '/' + target.title + if job_transfer.sub_path: + squonk_directory = os.path.join('/', settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path, target.title) + else: + squonk_directory = os.path.join('/', settings.SQUONK2_MEDIA_DIRECTORY, target.title) logger.info('+ Destination squonk_directory=%s', squonk_directory) # This to pick up NULL values from the changeover to using compounds. diff --git a/viewer/squonk_job_file_upload.py b/viewer/squonk_job_file_upload.py index 37ac7d34..6858536e 100644 --- a/viewer/squonk_job_file_upload.py +++ b/viewer/squonk_job_file_upload.py @@ -10,22 +10,22 @@ """ import json import os -import shutil -from django.conf import settings -from rest_framework import status from squonk2.dm_api import DmApi from celery.utils.log import get_task_logger -from viewer.models import JobRequest, User +from viewer.models import JobRequest from viewer.utils import ( SDF_VERSION, add_prop_to_sdf, create_media_sub_directory ) +from viewer.squonk2_agent import Squonk2Agent, get_squonk2_agent logger = get_task_logger(__name__) +_SQ2A: Squonk2Agent = get_squonk2_agent() + # A "Blank" molecule. # Inserted at the top of SDF files pulled from Squonk2. # This provides a 'header' that is then used @@ -69,7 +69,7 @@ def _insert_sdf_blank_mol(job_request, transition_time, sdf_filename): # Do nothing if the first line of the file matches the version we're about to set. blank_present = False - with open(sdf_filename, 'r') as in_file: + with open(sdf_filename, 'r', encoding='utf-8') as in_file: line = in_file.readline() if line and line.startswith(SDF_VERSION): blank_present = True @@ -79,7 +79,7 @@ def _insert_sdf_blank_mol(job_request, transition_time, sdf_filename): # Compound set reference URL. # What's the https-prefixed URL to the instance? # The record's URL is relative to the API. - ref_url = settings.SQUONK2_UI_URL + ref_url = _SQ2A.get_ui_url() if ref_url.endswith('/'): ref_url += job_request.squonk_url_ext else: @@ -103,9 +103,9 @@ def _insert_sdf_blank_mol(job_request, transition_time, sdf_filename): 'ref_url': ref_url} blank_mol = _SDF_BLANK_MOL_TEMPLATE.format(**variables) tmp_filename = sdf_filename + '.tmp' - with open(tmp_filename, 'w') as tmp_file: + with open(tmp_filename, 'w', encoding='utf-8') as tmp_file: tmp_file.write(blank_mol) - with open(sdf_filename, 'r') as in_file: + with open(sdf_filename, 'r', encoding='utf-8') as in_file: for line in in_file: tmp_file.write(line) os.remove(sdf_filename) @@ -138,17 +138,16 @@ def process_compound_set_file(jr_id, logger.info('Processing job compound file (%s)...', jr_id) + logger.info("Squonk transition_time='%s'", transition_time) + logger.info("Squonk job_output_path='%s'", job_output_path) + logger.info("Squonk job_output_filename='%s'", job_output_filename) + jr = JobRequest.objects.get(id=jr_id) - # The callback token (required to make Squonk API calls from the callback) - # and instance ID are in JobRequest. + # The callback token is required to make Squonk API calls from the callback context jr_job_info_msg = jr.squonk_job_info[1] - token = jr_job_info_msg.get('callback_token') - instance_id = jr_job_info_msg.get('instance_id') - logger.info("Squonk API token=%s", token) - logger.info("Squonk API instance_id=%s", instance_id) - - logger.info("Expecting Squonk path='%s'", job_output_path) + callback_token = jr_job_info_msg.get('callback_token') + logger.info("Squonk API callback_token=%s", callback_token) # Do we need to create the upload path? # This is used for this 'job' and is removed when the upload is complete @@ -182,7 +181,7 @@ def process_compound_set_file(jr_id, logger.info("Expecting Squonk job_output_param_filename='%s'...", job_output_param_filename) result = DmApi.get_unmanaged_project_file_with_token( - token=token, + token=callback_token, project_id=jr.squonk_project, project_path=job_output_path, project_file=job_output_param_filename, @@ -197,7 +196,7 @@ def process_compound_set_file(jr_id, logger.info("Expecting Squonk job_output_filename='%s'...", job_output_filename) result = DmApi.get_unmanaged_project_file_with_token( - token=token, + token=callback_token, project_id=jr.squonk_project, project_path=job_output_path, project_file=job_output_filename, @@ -209,12 +208,13 @@ def process_compound_set_file(jr_id, else: # Both files pulled back. got_all_files = True - # Delete the callback token, which is no-longer needed. - # Don't care if this fails - the token will expire automatically - # after a period of time. -# _ = DmApi.delete_instance_token( -# instance_id=instance_id, -# token=token) +# if instance_id: +# # Delete the callback token, which is no-longer needed. +# # Don't care if this fails - the token will expire automatically +# # after a period of time. +# _ = DmApi.delete_instance_token( +# instance_id=instance_id, +# token=callback_token) if not got_all_files: logger.warning('Not processing. Either %s or %s is missing', @@ -263,7 +263,7 @@ def process_compound_set_file(jr_id, # We take every field as the key and the description as the value. # If anything goes wrong we erase the SD file and return params = {} - with open(tmp_param_filename, 'r') as param_file: + with open(tmp_param_filename, 'r', encoding='utf-8') as param_file: meta = json.loads(param_file.read()) if 'annotations' not in meta or len(meta['annotations']) == 0: logger.warning('Not processing. No annotations in %s', diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index f0484f97..1e301b10 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -8,31 +8,31 @@ import logging import datetime -from django.conf import settings +import shortuuid + from squonk2.dm_api import DmApi -from viewer.models import ( Target, +from viewer.models import ( Project, + Target, Snapshot, JobRequest, JobFileTransfer ) -from viewer.utils import get_https_host +from viewer.utils import create_squonk_job_request_url, get_https_host +from viewer.squonk2_agent import CommonParams, Squonk2Agent, get_squonk2_agent logger = logging.getLogger(__name__) +_SQ2A: Squonk2Agent = get_squonk2_agent() def check_squonk_active(request): - """Call a Squonk API to check that Squonk can be reached. + """Call the Squonk2 Agent to check that Squonk2 can be reached. """ - logger.info('+ Ping') - auth_token = request.session['oidc_access_token'] - - logger.info('oidc token') - logger.info(auth_token) + logger.info('+ Squonk2Agent.ping()') - result = DmApi.ping(auth_token) - logger.debug(result) + ping_rv = _SQ2A.ping() + logger.debug(ping_rv) - return result.success + return ping_rv.success def get_squonk_job_config(request, @@ -87,41 +87,72 @@ def create_squonk_job(request): a URL allowing the front end to link to the running job instance """ - logger.info('+ create_squonk_job') + logger.info('+ create_squonk_job()') auth_token = request.session['oidc_access_token'] logger.debug(auth_token) - squonk_job_name = request.data['squonk_job_name'] + access_id = request.data['access'] # The access ID/legacy Project record title target_id = request.data['target'] snapshot_id = request.data['snapshot'] - squonk_project = request.data['squonk_project'] + session_project_id = request.data['session_project'] + squonk_job_name = request.data['squonk_job_name'] squonk_job_spec = request.data['squonk_job_spec'] - logger.info('+ squonk_job_name=%s', squonk_job_name) + logger.info('+ access_id=%s', access_id) logger.info('+ target_id=%s', target_id) logger.info('+ snapshot_id=%s', snapshot_id) - logger.info('+ squonk_project=%s', squonk_project) + logger.info('+ session_project_id=%s', session_project_id) + logger.info('+ squonk_job_name=%s', squonk_job_name) logger.info('+ squonk_job_spec=%s', squonk_job_spec) job_transfers = JobFileTransfer.objects.filter(snapshot=snapshot_id) if not job_transfers: - logger.warning('No JobFileTransfer object for snapshot %s', snapshot_id) - raise ValueError('No JobFileTransfer object for snapshot %s.' - ' Files must be transferred before a job can be requested.', - snapshot_id) + logger.warning('+ create_squonk_job() - No JobFileTransfer object for snapshot %s', snapshot_id) + raise ValueError(f'No JobFileTransfer object for snapshot {snapshot_id}.' + ' Files must be transferred before a job can be requested.') job_transfer = JobFileTransfer.objects.filter(snapshot=snapshot_id).latest('id') if job_transfer.transfer_status != 'SUCCESS': - logger.warning('JobFileTransfer status is not SUCCESS (is %s)', + logger.warning('+ create_squonk_job() - JobFileTransfer status is not SUCCESS (is %s)', job_transfer.transfer_status) raise ValueError('Job Transfer not complete') + logger.info('+ create_squonk_job() calling ensure_project() to get the Squonk2 Project') + + # This requires a Squonk2 Project (created by the Squonk2Agent). + # It may be an existing project, or it might be a new project. + user = request.user + common_params = CommonParams(user_id=user.id, + access_id=access_id, + target_id=target_id, + session_id=session_project_id) + sq2_rv = _SQ2A.ensure_project(common_params) + if not sq2_rv.success: + msg = f'+ create_squonk_job() - JobTransfer failed to get/create a Squonk2 Project' \ + f' for User "{user.username}", Access ID {access_id},' \ + f' Target ID {target_id}, and SessionProject ID {session_project_id}.' \ + f' Got "{sq2_rv.msg}".' \ + ' Cannot continue' + logger.error(msg) + raise ValueError(msg) + + # The Squonk2Project record is in the response msg + squonk2_project_uuid = sq2_rv.msg.uuid + squonk2_unit_name = sq2_rv.msg.unit.name + squonk2_unit_uuid = sq2_rv.msg.unit.uuid + logger.info('+ create_squonk_job() ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)', + squonk2_project_uuid, squonk2_unit_name, squonk2_unit_uuid) + job_request = JobRequest() job_request.squonk_job_name = squonk_job_name job_request.user = request.user + job_request.project = Project.objects.get(id=access_id) job_request.snapshot = Snapshot.objects.get(id=snapshot_id) job_request.target = Target.objects.get(id=target_id) - job_request.squonk_project = squonk_project + # We should use a foreign key, + # but to avoid migration issues with the existing code + # we continue to use the project UUID string field. + job_request.squonk_project = squonk2_project_uuid job_request.squonk_job_spec = squonk_job_spec # Saving creates the uuid for the callback @@ -134,32 +165,70 @@ def create_squonk_job(request): # Used for identifying the run, set to the username + date. job_name = job_request.user.username + '-' + datetime.date.today().strftime('%Y-%m-%d') - logger.info('+ job_name=%s', job_name) - logger.info('+ callback_url=%s', callback_url) - logger.info('+ Calling DmApi.start_job_instance(%s)', job_name) + # Create a callback token + # that we can use on the job from our callback context. + # It is required to be a shortuuid of 22 characters using the default character set + callback_token = shortuuid.uuid() + + logger.info('+ create_squonk_job() job_name=%s', job_name) + logger.info('+ create_squonk_job(%s) callback_url=%s', job_name, callback_url) + logger.info('+ create_squonk_job(%s) callback_token=%s', job_name, callback_token) + + # Dry-run the Job execution (this provides us with the 'command', which is + # placed in the JobRecord's squonk_job_info). + logger.info('+ create_squonk_job(%s) code=%s calling DmApi.dry_run_job_instance()', + job_name, job_request.code) + result = DmApi.dry_run_job_instance(auth_token, + project_id=job_request.squonk_project, + name=job_name, + callback_url=callback_url, + callback_token=callback_token, + callback_context=job_request.code, + specification=json.loads(squonk_job_spec)) + logger.debug(result) + + if not result.success: + logger.warning('+ create_squonk_job(%s) code=%s dry_run_job_instance() FAILED result=%s', + job_name, job_request.code, result) + job_request.delete() + raise ValueError(result.msg) + + # We can now commit the JobRequest record so that it's ready + # for use by any callbacks. The 'result' will contain the callback token + # and the Job's decoded command (that will be interrogated when the Job s complete) + job_request.squonk_job_info = result + job_request.job_start_datetime = datetime.datetime.utcnow() + job_request.save() + + # Now start the job 'for real'... + logger.info('+ create_squonk_job(%s) code=%s calling DmApi.start_job_instance()', + job_name, job_request.code) result = DmApi.start_job_instance(auth_token, project_id=job_request.squonk_project, name=job_name, callback_url=callback_url, - generate_callback_token=True, + callback_token=callback_token, + callback_context=job_request.code, specification=json.loads(squonk_job_spec), timeout_s=8) logger.debug(result) if not result.success: - logger.warning('+ start_job_instance(%s) result=%s', job_name, result) - logger.error('+ FAILED (job probably did not start) (%s)', job_name) + logger.warning('+ create_squonk_job(%s) code=%s result=%s', + job_name, job_request.code, result) + logger.error('+ start_job_instance(%s) code=%s FAILED (job probably did not start)', + job_name, job_request.code) job_request.delete() raise ValueError(result.msg) - job_request.squonk_job_info = result - job_request.squonk_url_ext = settings.SQUONK2_INSTANCE_API + str(result.msg['instance_id']) - job_request.job_start_datetime = datetime.datetime.utcnow() - job_request.save() - job_instance_id = result.msg['instance_id'] job_task_id = result.msg['task_id'] - logger.info('+ SUCCESS. Job "%s" started (job_instance_id=%s job_task_id=%s)', - job_name, job_instance_id, job_task_id) + logger.info('+ create_squonk_job(%s) code=%s SUCCESS (job_instance_id=%s job_task_id=%s)', + job_name, job_request.code, job_instance_id, job_task_id) + + # Manufacture the Squonk URL (actually set in the callback) + # so the front-end can use it immediately (we cannot set the JobRequest + # `squonk_url_ext` here as it introduces a race-condition with the callback logic). + squonk_url_ext = create_squonk_job_request_url(job_instance_id) - return job_request.id, job_request.squonk_url_ext + return job_request.id, squonk_url_ext diff --git a/viewer/target_set_upload.py b/viewer/target_set_upload.py index d48c87a4..e5eb55e0 100644 --- a/viewer/target_set_upload.py +++ b/viewer/target_set_upload.py @@ -6,7 +6,7 @@ functions.py """ import logging -import sys, json, os, glob, shutil +import sys, json, os, shutil import datetime from django.contrib.auth.models import User @@ -172,7 +172,7 @@ def add_prot(code, target, xtal_path, xtal, input_dict): """Add a protein with a PDB, code and :param code: the unique code for this file - :param target: the target to be linkede to + :param target: the target to be linked to :param xtal_path: the path to the crystal directory :param xtal: name of the xtal(?) :param input_dict: dictionary of files in crystal directory from load_dir @@ -183,13 +183,13 @@ def add_prot(code, target, xtal_path, xtal, input_dict): # code is normally the xtal directory in the aligned folder, but this may have been modified to have # an alternate name added to it - in the form 'directory:alternate_name'. code_first_part = code.split(":")[0] - proteins = Protein.objects.filter(code__contains=code_first_part) + proteins = Protein.objects.filter(code__contains=code_first_part, target_id=target) if proteins.exists(): new_prot = proteins.first() - logger.debug("Protein exists='%s'", new_prot[1]) + logger.debug("Pre-existing Protein (%s)", new_prot) else: new_prot = Protein.objects.get_or_create(code=code, target_id=target) - logger.debug("Protein created new_prot='%s'", new_prot[1]) + logger.debug("New Protein (code='%s' target_id='%s')", code, target) new_prot = new_prot[0] new_prot.apo_holo = True @@ -332,7 +332,7 @@ def add_mol(mol_file, prot, projects, lig_id="LIG", chaind_id="Z", """ # create mol object from mol_sd rd_mol = Chem.MolFromMolFile(mol_file) - orig_mol_block = open(mol_file, 'r').read() + orig_mol_block = open(mol_file, 'r', encoding='utf-8').read() if rd_mol is None: return None @@ -374,7 +374,7 @@ def add_mol(mol_file, prot, projects, lig_id="LIG", chaind_id="Z", if sdf_file: new_mol.sdf_file.save( os.path.basename(sdf_file), - File(open(sdf_file)) + File(open(sdf_file, encoding='utf-8')) ) new_mol.save() return new_mol @@ -462,7 +462,7 @@ def add_map(new_prot, new_target, map_path, map_type): hotspot_map = HotspotMap.objects.get_or_create( map_type=map_type, target_id=new_target, prot_id=new_prot )[0] - hotspot_map.map_info.save(os.path.basename(map_path), File(open(map_path))) + hotspot_map.map_info.save(os.path.basename(map_path), File(open(map_path, encoding='utf-8'))) return hotspot_map @@ -477,7 +477,7 @@ def delete_users(project): project.save() -def get_create_projects(target, proposal_ref): +def get_create_projects(target, proposal_ref, proposal_code='lb'): """Add proposals and visits as projects for a given target. :param new_target: the target being added @@ -494,6 +494,10 @@ def get_create_projects(target, proposal_ref): # The first word is the ISPY proposal/visit name that is used as the title of the project. # It can be set to OPEN in which case there are no users. visit = proposal_ref.split()[0] + # If the visit is not prefixed by the proposal code + # (typically a 2-letter sequence like "lb") then prefix it. + if visit[0].isdigit(): + visit = f"{proposal_code}{visit}" project = Project.objects.get_or_create(title=visit)[0] projects.append(project) @@ -504,9 +508,14 @@ def get_create_projects(target, proposal_ref): target.project_id.add(project) # Remaining words in proposal_ref (if any) must be fedid's which are used to find users information. + num_users = 0 for fedid in proposal_ref.split()[1:]: user = User.objects.get_or_create(username=fedid, password="")[0] project.user_id.add(user) + num_users += 1 + if num_users == 0: + project.open_to_public = True + target.upload_progess = 10.00 target.save() @@ -537,7 +546,7 @@ def remove_not_added(target, xtal_list): def save_confidence(mol, file_path, annotation_type="ligand_confidence"): """save ligand confidence""" - input_dict = json.load(open(file_path)) + input_dict = json.load(open(file_path), encoding='utf-8') val_store_dict = ["ligand_confidence_comment", "refinement_outcome", "ligand_confidence_int"] for val in val_store_dict: if val in input_dict: @@ -926,7 +935,7 @@ def analyse_target(target, aligned_path): new_frame.sort_values(by='site_name', inplace=True) # one file for new names - with open(os.path.join(aligned_path, 'alternate_names.csv'), 'a') as f: + with open(os.path.join(aligned_path, 'alternate_names.csv'), 'a', encoding='utf-8') as f: f.write('name,alternate_name\n') for _, row in new_frame.iterrows(): @@ -945,7 +954,7 @@ def analyse_target(target, aligned_path): for i in range(0, len(sorted(unique_sites))): site_mapping[unique_sites[i]] = i - with open(os.path.join(aligned_path, 'hits_ids.csv'), 'a') as f: + with open(os.path.join(aligned_path, 'hits_ids.csv'), 'a', encoding='utf-8') as f: f.write('crystal_id,site_number\n') for _, row in new_frame.iterrows(): @@ -956,7 +965,7 @@ def analyse_target(target, aligned_path): for crys in list(set([c.code for c in crystal])): f.write(str(crys) + ',' + str(s_id) + '\n') - with open(os.path.join(aligned_path, 'sites.csv'), 'a') as f: + with open(os.path.join(aligned_path, 'sites.csv'), 'a', encoding='utf-8') as f: f.write('site,id\n') for key in site_mapping.keys(): f.write(str(key) + ',' + str(site_mapping[key]) + '\n') @@ -1173,25 +1182,45 @@ def validate_target(new_data_folder, target_name, proposal_ref): :param proposal_ref: A reference to the proposal/visit used for connecting the target to a project/users :return: """ + # Don't need + del proposal_ref + validate_dict = {'Location': [], 'Error': [], 'Line number': []} # Check if there is any data to process target_path = os.path.join(new_data_folder, target_name) - if not os.path.isdir(target_path): - validate_dict = add_tset_warning(validate_dict, 'Folder', f'No folder matching target name in extracted zip file {new_data_folder}, {target_name}', 0) - - aligned_path = os.path.join(target_path, 'aligned') - - if not os.path.isdir(aligned_path): - validate_dict = add_tset_warning(validate_dict, 'Folder', 'No aligned folder present in target name folder', 0) + # Assume success... + validated = True - # Check if there is a metadata.csv file to process - metadata_file = os.path.join(aligned_path, 'metadata.csv') - if os.path.isfile(metadata_file): - validated, validate_dict = check_metadata(metadata_file, validate_dict) - else: - validate_dict = add_tset_warning(validate_dict, 'File', 'No metedata.csv file present in the aligned folder', 0) + # A target directory must exist + if not os.path.isdir(target_path): + validate_dict = add_tset_warning(validate_dict, 'Folder', + 'Folder does not match target name.' + f' Expected "{target_name}".' + f' Is the upload called "{target_name}.zip"?', 0) + # No point in checking anything else if this check fails validated = False + if validated: + # An 'aligned' directory must exist + aligned_path = os.path.join(target_path, 'aligned') + if not os.path.isdir(aligned_path): + validate_dict = add_tset_warning(validate_dict, 'Folder', + 'No aligned folder present.' + f' Expected "{target_name}/{aligned_path}"', 0) + # No point in checking anything else if this check fails + ok_so_far = False + + if validated: + # A metadata.csv file must exist + metadata_file = os.path.join(aligned_path, 'metadata.csv') + if os.path.isfile(metadata_file): + validated, validate_dict = check_metadata(metadata_file, validate_dict) + else: + validate_dict = add_tset_warning(validate_dict, 'File', + 'No metedata file present.' + f' Expected "{target_name}/{aligned_path}/{metadata_file}"', 0) + validated = False + return validated, validate_dict diff --git a/viewer/tasks.py b/viewer/tasks.py index 96c05039..5141193e 100644 --- a/viewer/tasks.py +++ b/viewer/tasks.py @@ -7,7 +7,6 @@ import django django.setup() -from django.conf import settings from celery import shared_task import numpy as np @@ -112,6 +111,7 @@ def process_compound_set(validate_output): process_stage, process_type, validate_dict, validated, params = validate_output logger.info('process_compound_set() ENTER') + logger.info('process_compound_set() process_type=%s', process_type) logger.info('process_compound_set() validated=%s', validated) logger.info('process_compound_set() params=%s', params) @@ -244,10 +244,6 @@ def validate_compound_set(task_params): # print('%d mols detected (including blank mol)' % (len(suppl),)) blank_mol = suppl[0] - # Get submitter name/info for passing into upload to get unique name - submitter_name = blank_mol.GetProp('submitter_name') - submitter_method = blank_mol.GetProp('method') - if blank_mol is None: validate_dict = add_warning(molecule_name='Blank Mol', field='N/A', @@ -260,6 +256,9 @@ def validate_compound_set(task_params): logger.warning('validate_compound_set() EXIT' ' user_id=%s sdf_file=%s validated=False', user_id, sdf_file) + # Can't get submitter name or method when there is now mol + submitter_name = '' + submitter_method = '' return (validate_dict, validated, sdf_file, target, zfile, submitter_name, submitter_method) @@ -295,7 +294,10 @@ def validate_compound_set(task_params): props = [key for key in list(mol.GetPropsAsDict().keys())] diff_list = np.setdiff1d(props, unique_props) for diff in diff_list: - add_warning(molecule_name=mol.GetProp('_Name'), + molecule_name = 'Unknown (no _Name property)' + if mol.HasProp('_Name'): + molecule_name = mol.GetProp('_Name') + add_warning(molecule_name=molecule_name, field='property (missing)', warning_string=f'{diff} property is missing from this molecule', validate_dict=validate_dict) @@ -321,7 +323,10 @@ def validate_compound_set(task_params): for m in other_mols: if m: validate_dict = check_mol_props(m, validate_dict) - validate_dict = check_name_characters(m.GetProp('_Name'), validate_dict) + molecule_name = '' + if m.HasProp('_Name'): + molecule_name = m.GetProp('_Name') + validate_dict = check_name_characters(molecule_name, validate_dict) # validate_dict = check_pdb(m, validate_dict, target, zfile) validate_dict = check_refmol(m, validate_dict, target) validate_dict = check_field_populated(m, validate_dict) @@ -474,7 +479,7 @@ def validate_target_set(target_zip, target=None, proposal=None, email=None): - submitter_name (str): name of the user who submitted the upload """ - logger.info('+ validating target set: ' + target_zip) + logger.info('+ TASK target_zip=%s', target_zip) # Get submitter name/info for passing into upload to get unique name submitter_name = '' @@ -554,7 +559,7 @@ def process_job_file_transfer(auth_token, id): """ - logger.info('+ Starting File Transfer (%s) [STARTED]', id) + logger.info('+ TASK Starting File Transfer (%s) [STARTED]', id) job_transfer = JobFileTransfer.objects.get(id=id) job_transfer.transfer_status = "STARTED" job_transfer.transfer_task_id = str(process_job_file_transfer.request.id) @@ -562,11 +567,11 @@ def process_job_file_transfer(auth_token, id): try: process_file_transfer(auth_token, job_transfer.id) except RuntimeError as error: - logger.error('- File Transfer failed %s', id) + logger.error('- TASK File transfer (%s) [FAILED] error=%s', + id, error) logger.error(error) job_transfer.transfer_status = "FAILURE" job_transfer.save() - logger.info('+ Failed File Transfer (%s) [FAILURE]', id) else: # Update the transfer datetime for comparison with the target upload datetime. # This should only be done on a successful upload. @@ -577,7 +582,7 @@ def process_job_file_transfer(auth_token, id): "compounds": list(SQUONK_COMP_MAPPING.keys())} job_transfer.transfer_spec = files_spec job_transfer.save() - logger.info('+ Successful File Transfer (%s) [SUCCESS]', id) + logger.info('+ TASK File transfer (%s) [SUCCESS]', id) return job_transfer.transfer_status @@ -606,7 +611,8 @@ def process_compound_set_job_file(task_params): job_output_path = task_params['job_output_path'] job_output_filename = task_params['job_output_filename'] - logger.info('+ Starting File Upload (%s)', jr_id) + logger.info('+ TASK task_params=%s', task_params) + job_request = JobRequest.objects.get(id=jr_id) job_request.upload_task_id = str(process_compound_set_job_file.request.id) job_request.save() @@ -618,12 +624,12 @@ def process_compound_set_job_file(task_params): job_output_path, job_output_filename) except RuntimeError as error: - logger.error('- File Upload failed (%s)', jr_id) + logger.error('- TASK file Upload failed (%s)', jr_id) logger.error(error) else: # Update the transfer datetime for comparison with the target upload datetime. # This should only be done on a successful upload. - logger.info('- File Upload Ended Successfully (%s)', jr_id) + logger.info('+ TASK file Upload Ended Successfully (%s)', jr_id) # We are expected to be followed by 'validate_compound_set' # which expects user_id, sdf_file and target @@ -655,6 +661,7 @@ def erase_compound_set_job_material(task_params, job_request_id=0): return job_request = JobRequest.objects.get(id=job_request_id) + logger.info('+ TASK Erasing job material job_request %s', job_request) # Upload done (successfully or not) # 'task_params' (a dictionary) is expected to be @@ -677,13 +684,13 @@ def erase_compound_set_job_material(task_params, job_request_id=0): job_request.computed_set = cs else: # Failed validation? - logger.info('Upload failed (%d) - task_params=%s', - job_request_id, task_params) + logger.info('- TASK Upload failed (%d) - task_params=%s', + job_request_id, task_params) logger.warning('Upload failed (%d) - process_stage value not satisfied', job_request_id) job_request.upload_status = 'FAILURE' job_request.save() - logger.info('Updated job_request %s', job_request) + logger.info('+ TASK Erased and updated job_request %s', job_request) # Always erase uploaded data delete_media_sub_directory(get_upload_sub_directory(job_request)) diff --git a/viewer/urls.py b/viewer/urls.py index 1a0c3b1c..ce8a84c4 100644 --- a/viewer/urls.py +++ b/viewer/urls.py @@ -17,4 +17,5 @@ url(r'^protein_set/(?P.+)/$', views.pset_download, name='protein_set'), url(r'^target/(?P.+)/$', views.tset_download, name='target_set'), url(r'upload_designs/', views.DSetUploadView.as_view(), name='upload_designs'), + url(r"job_access/", views.JobAccessView.as_view(), name='job_access'), ] diff --git a/viewer/utils.py b/viewer/utils.py index 34199b83..3471331a 100644 --- a/viewer/utils.py +++ b/viewer/utils.py @@ -16,6 +16,12 @@ SDF_VERSION = 'ver_1.2' +def create_squonk_job_request_url(instance_id): + """Creates the Squonk Instance API url from an instance ID (UUID). + """ + return settings.SQUONK2_INSTANCE_API + str(instance_id) + + def create_media_sub_directory(sub_directory): """Creates a directory (or directories) in the MEDIA directory, returning the full path. @@ -61,8 +67,8 @@ def add_prop_to_sdf(sdf_file_in, sdf_file_out, properties): _REC_SEPARATOR = '$$$$\n' found_separator = False - with open(sdf_file_out, 'a') as sdf_out: - with open(sdf_file_in, 'r') as sdf_in: + with open(sdf_file_out, 'a', encoding='utf-8') as sdf_out: + with open(sdf_file_in, 'r', encoding='utf-8') as sdf_in: while True: line = sdf_in.readline() if line: @@ -133,4 +139,3 @@ def get_https_host(request): Note that this link will not work on local """ return settings.SECURE_PROXY_SSL_HEADER[1] + '://' + request.get_host() - diff --git a/viewer/views.py b/viewer/views.py index 0d392787..db66df15 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -1,7 +1,9 @@ +import logging import json import os import zipfile from io import StringIO +import re import uuid import shlex import shutil @@ -10,8 +12,7 @@ from dateutil.parser import parse import pytz -# import the logging library -import logging +import shortuuid import pandas as pd from django.db import connections @@ -24,22 +25,24 @@ from django.http import JsonResponse from django.views import View -from rest_framework import viewsets -from rest_framework.parsers import BaseParser +from rest_framework import status, viewsets from rest_framework.exceptions import ParseError -from rest_framework.views import APIView +from rest_framework.parsers import BaseParser +from rest_framework.renderers import JSONRenderer from rest_framework.response import Response -from rest_framework import status +from rest_framework.views import APIView from celery.result import AsyncResult from api.security import ISpyBSafeQuerySet -from api.utils import get_params, get_highlighted_diffs +from api.utils import get_params, get_highlighted_diffs, pretty_request +from viewer.utils import create_squonk_job_request_url from viewer.models import ( Molecule, Protein, + Project, Compound, Target, ActionType, @@ -49,10 +52,8 @@ SnapshotActions, ComputedMolecule, ComputedSet, - CSetKeys, NumericalScoreValues, ScoreDescription, - File, TagCategory, TextScoreValues, MoleculeTag, @@ -62,7 +63,10 @@ JobFileTransfer ) from viewer import filters -from .forms import CSetForm, CSetUpdateForm, TSetForm +from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent +from viewer.squonk2_agent import AccessParams, CommonParams, SendParams, RunJobParams + +from .forms import CSetForm, TSetForm from .tasks import ( check_services, erase_compound_set_job_material, @@ -96,11 +100,11 @@ ProteinSerializer, CompoundSerializer, TargetSerializer, - MolImageSerialzier, - CmpdImageSerialzier, - ProtMapInfoSerialzer, - ProtPDBInfoSerialzer, - ProtPDBBoundInfoSerialzer, + MolImageSerializer, + CmpdImageSerializer, + ProtMapInfoSerializer, + ProtPDBInfoSerializer, + ProtPDBBoundInfoSerializer, VectorsSerializer, GraphSerializer, ActionTypeSerializer, @@ -110,7 +114,6 @@ SnapshotReadSerializer, SnapshotWriteSerializer, SnapshotActionsSerializer, - FileSerializer, ComputedSetSerializer, ComputedMoleculeSerializer, NumericalScoreSerializer, @@ -129,7 +132,8 @@ JobRequestReadSerializer, JobRequestWriteSerializer, JobCallBackReadSerializer, - JobCallBackWriteSerializer + JobCallBackWriteSerializer, + ProjectSerializer, ) logger = logging.getLogger(__name__) @@ -141,6 +145,7 @@ _SESSION_ERROR = 'session_error' _SESSION_MESSAGE = 'session_message' +_SQ2A: Squonk2Agent = get_squonk2_agent() class VectorsView(ISpyBSafeQuerySet): """ DjagnoRF view for vectors @@ -274,7 +279,7 @@ class MolImageView(ISpyBSafeQuerySet): "mol_image": "<?xml version='1.0' encoding='iso-8859-1'?><svg version='1.1' nk'..."}] """ queryset = Molecule.objects.filter() - serializer_class = MolImageSerialzier + serializer_class = MolImageSerializer filter_permissions = "prot_id__target_id__project_id" filterset_fields = ("prot_id", "cmpd_id", "smiles", "prot_id__target_id", "mol_groups") @@ -303,7 +308,7 @@ class CompoundImageView(ISpyBSafeQuerySet): """ queryset = Compound.objects.filter() - serializer_class = CmpdImageSerialzier + serializer_class = CmpdImageSerializer filter_permissions = "project_id" filterset_fields = ("smiles",) @@ -326,7 +331,7 @@ class ProteinMapInfoView(ISpyBSafeQuerySet): """ queryset = Protein.objects.filter() - serializer_class = ProtMapInfoSerialzer + serializer_class = ProtMapInfoSerializer filter_permissions = "target_id__project_id" filterset_fields = ("code", "target_id", "target_id__title", "prot_type") @@ -362,7 +367,7 @@ class ProteinPDBInfoView(ISpyBSafeQuerySet): """ queryset = Protein.objects.filter() - serializer_class = ProtPDBInfoSerialzer + serializer_class = ProtPDBInfoSerializer filter_permissions = "target_id__project_id" filterset_fields = ("code", "target_id", "target_id__title", "prot_type") @@ -398,11 +403,47 @@ class ProteinPDBBoundInfoView(ISpyBSafeQuerySet): """ queryset = Protein.objects.filter() - serializer_class = ProtPDBBoundInfoSerialzer + serializer_class = ProtPDBBoundInfoSerializer filter_permissions = "target_id__project_id" filterset_fields = ("code", "target_id", "target_id__title", "prot_type") +class ProjectView(ISpyBSafeQuerySet): + """ DjagnoRF view to retrieve info about projects + + Methods + ------- + url: + api/projects + queryset: + `viewer.models.Project.objects.filter()` + returns: JSON + - id: id of the project object + - title: name of the target + - init_date: The date the Project was created + + example output: + + .. code-block:: javascript + + "results": [ + { + "id": 2, + "target_access_string": "lb27156-1", + "init_date": "2023-01-09T15:00:00Z", + "authority": "DIAMOND-ISPYB", + "open_to_public": false, + "user_can_use_squonk": false + } + ] + + """ + queryset = Project.objects.filter() + serializer_class = ProjectSerializer + # Special case - Project filter permissions is blank. + filter_permissions = "" + + class TargetView(ISpyBSafeQuerySet): """ DjagnoRF view to retrieve info about targets @@ -641,19 +682,25 @@ def react(request): """ discourse_api_key = settings.DISCOURSE_API_KEY - squonk_api_url = settings.SQUONK2_DMAPI_URL - squonk_ui_url = settings.SQUONK2_UI_URL context = {} - context['discourse_available'] = 'false' - context['squonk_available'] = 'false' + + # Is the Squonk2 Agent configured? + logger.info("Checking whether Squonk2 is configured...") + sq2_rv = _SQ2A.configured() + if sq2_rv.success: + logger.info("Squonk2 is configured") + context['squonk_available'] = 'true' + else: + logger.info("Squonk2 is NOT configured") + context['squonk_available'] = 'false' + if discourse_api_key: context['discourse_available'] = 'true' - if squonk_api_url and squonk_ui_url: - context['squonk_available'] = 'true' + else: + context['discourse_available'] = 'false' user = request.user - if user.is_authenticated: context['discourse_host'] = '' context['user_present_on_discourse'] = 'false' @@ -668,8 +715,8 @@ def react(request): # If user is authenticated Squonk can be called then return the Squonk host # so the Frontend can navigate to it context['squonk_ui_url'] = '' - if squonk_api_url and check_squonk_active(request): - context['squonk_ui_url'] = settings.SQUONK2_UI_URL + if sq2_rv.success and check_squonk_active(request): + context['squonk_ui_url'] = _SQ2A.get_ui_url() return render(request, "viewer/react_temp.html", context) @@ -728,7 +775,7 @@ def save_tmp_file(myfile): return tmp_file -class UploadCSet(View): +class UploadCSet(APIView): """ View to render and control viewer/upload-cset.html - a page allowing upload of computed sets. Validation and upload tasks are defined in `viewer.compound_set_upload`, `viewer.sdf_check` and `viewer.tasks` and the task response handling is done by `viewer.views.ValidateTaskView` and `viewer.views.UploadTaskView` @@ -758,6 +805,11 @@ class UploadCSet(View): def get(self, request): + tag = '+ UploadCSet GET' + logger.info('%s', pretty_request(request, tag=tag)) + logger.info('User=%s', str(request.user)) +# logger.info('Auth=%s', str(request.auth)) + # Any messages passed to us via the session? # Maybe from a redirect? # It so take them and remove them. @@ -790,6 +842,11 @@ def get(self, request): def post(self, request): + tag = '+ UploadCSet POST' + logger.info('%s', pretty_request(request, tag=tag)) + logger.info('User=%s', str(request.user)) +# logger.info('Auth=%s', str(request.auth)) + # Only authenticated users can upload files # - this can be switched off in settings.py. user = self.request.user @@ -805,7 +862,6 @@ def post(self, request): assert check_services() form = CSetForm(request.POST, request.FILES) - if form.is_valid(): # Get all the variables needed from the form. @@ -819,18 +875,21 @@ def post(self, request): # The 'sdf_file' anf 'target_name' are only required for upload/update sdf_file = request.FILES.get('sdf_file') target = request.POST.get('target_name') - update_set = request.POST['update_set'] + update_set = request.POST.get('update_set') + logger.info('+ UploadCSet POST choice="%s" target="%s" update_set="%s"', choice, target, update_set) + # If a set is named the ComputedSet cannot be 'Anonymous' # and the user has to be the owner. selected_set = None - if update_set != 'None': + if update_set and update_set != 'None': computed_set_query = ComputedSet.objects.filter(unique_name=update_set) if computed_set_query: selected_set = computed_set_query[0] else: request.session[_SESSION_ERROR] = \ 'The set could not be found' + logger.warning('- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]) return redirect('upload_cset') # If validating or uploading we need a Target and SDF file. @@ -840,11 +899,13 @@ def post(self, request): request.session[_SESSION_ERROR] = \ 'To Validate or Upload' \ ' you must provide a Target and SDF file' + logger.warning('- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]) return redirect('upload_cset') elif choice in ['D']: if update_set == 'None': request.session[_SESSION_ERROR] = \ 'To Delete you must select an existing set' + logger.warning('- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]) return redirect('upload_cset') # If uploading (updating) or deleting @@ -860,6 +921,7 @@ def post(self, request): # Something wrong? # If so redirect... if _SESSION_ERROR in request.session: + logger.warning('- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]) return redirect('upload_cset') # Save uploaded sdf and zip to tmp storage @@ -883,6 +945,8 @@ def post(self, request): task_params['update'] = update_set task_validate = validate_compound_set.delay(task_params) + logger.info('+ UploadCSet POST "Validate" task underway') + # Update client side with task id and status context = {'validate_task_id': task_validate.id, 'validate_task_status': task_validate.status} @@ -903,6 +967,8 @@ def post(self, request): validate_compound_set.s(task_params) | process_compound_set.s()).apply_async() + logger.info('+ UploadCSet POST "Upload" task underway') + # Update client side with task id and status context = {'upload_task_id': task_upload.id, 'upload_task_status': task_upload.status} @@ -914,7 +980,18 @@ def post(self, request): request.session[_SESSION_MESSAGE] = \ f'Compound set "{selected_set.unique_name}" deleted' + + logger.info('+ UploadCSet POST "Delete" done') + return redirect('upload_cset') + + else: + logger.warning('+ UploadCSet POST unsupported submit_choice value (%s)', choice) + + else: + logger.warning('- UploadCSet POST form.is_valid() returned False') + + logger.info('- UploadCSet POST (leaving)') context = {'form': form} return render(request, 'viewer/upload-cset.html', context) @@ -922,7 +999,7 @@ def post(self, request): # Upload Target datasets functions -class UploadTSet(View): +class UploadTSet(APIView): """ View to render and control viewer/upload-tset.html - a page allowing upload of computed sets. Validation and upload tasks are defined in `viewer.target_set_upload`, `viewer.sdf_check` and `viewer.tasks` and the task response handling is done by `viewer.views.ValidateTaskView` and `viewer.views.UploadTaskView` @@ -951,6 +1028,11 @@ class UploadTSet(View): def get(self, request): + tag = '+ UploadTSet GET' + logger.info('%s', pretty_request(request, tag=tag)) + logger.info('User="%s"', str(request.user)) +# logger.info('Auth="%s"', str(request.auth)) + # Only authenticated users can upload files - this can be switched off in settings.py. user = self.request.user if not user.is_authenticated and settings.AUTHENTICATE_UPLOAD: @@ -968,7 +1050,12 @@ def get(self, request): return render(request, 'viewer/upload-tset.html', {'form': form}) def post(self, request): - logger.info('+ UploadTSet.post') + + tag = '+ UploadTSet POST' + logger.info('%s', pretty_request(request, tag=tag)) + logger.info('User="%s"', str(request.user)) +# logger.info('Auth="%s"', str(request.auth)) + context = {} # Only authenticated users can upload files - this can be switched off in settings.py. @@ -1054,10 +1141,10 @@ def email_task_completion(contact_email, message_type, target_name, target_path= 'Please navigate the following link to check the errors: validate_task/' + str(task_id) recipient_list = [contact_email, ] - logger.info('+ email_notify_task_completion email_from: ' + email_from ) - logger.info('+ email_notify_task_completion subject: ' + subject ) - logger.info('+ email_notify_task_completion message: ' + message ) - logger.info('+ email_notify_task_completion contact_email: ' + contact_email ) + logger.info('+ email_notify_task_completion email_from: %s', email_from ) + logger.info('+ email_notify_task_completion subject: %s', subject ) + logger.info('+ email_notify_task_completion message: %s', message ) + logger.info('+ email_notify_task_completion contact_email: %s', contact_email ) # Send email - this should not prevent returning to the screen in the case of error. send_mail(subject, message, email_from, recipient_list, fail_silently=True) @@ -1276,7 +1363,7 @@ def get(self, request, upload_task_id): target_path = '/viewer/target/%s' % target_name response_data['results'] = {} response_data['results']['tset_download_url'] = target_path - logger.info('+ UploadTaskView.get.success -email:'+contact_email) + logger.info('+ UploadTaskView.get.success -email: %s', contact_email) email_task_completion(contact_email, 'upload-success', target_name, target_path=target_path) else: cset_name = results[2] @@ -1433,7 +1520,7 @@ def cset_download(request, name): """ compound_set = ComputedSet.objects.get(unique_name=name) filepath = compound_set.submitted_sdf - with open(filepath.path, 'r') as fp: + with open(filepath.path, 'r', encoding='utf-8') as fp: data = fp.read() filename = 'compund-set_' + name + '.sdf' response = HttpResponse(content_type='text/plain') @@ -1472,7 +1559,7 @@ def pset_download(request, name): zip_obj = zipfile.ZipFile(buff, 'w') for fp in pdb_filepaths: - data = open(fp, 'r').read() + data = open(fp, 'r', encoding='utf-8').read() zip_obj.writestr(fp.split('/')[-1], data) zip_obj.close() @@ -1894,6 +1981,8 @@ class DSetUploadView(APIView): def put(self, request, format=None): """Method to handle PUT request and upload a design set """ + # Don't need... + del format f = request.FILES['file'] set_type = request.PUT['type'] @@ -2350,7 +2439,7 @@ def create(self, request): logger.info('+ DiscoursePostView.post') data = request.data - logger.info('+ DiscoursePostView.post'+json.dumps(data)) + logger.info('+ DiscoursePostView.post %s', json.dumps(data)) if data['category_name'] == '': category_details = None else: @@ -2379,7 +2468,7 @@ def list(self, request): """ logger.info('+ DiscoursePostView.get') query_params = request.query_params - logger.info('+ DiscoursePostView.get'+json.dumps(query_params)) + logger.info('+ DiscoursePostView.get %s', json.dumps(query_params)) discourse_api_key = settings.DISCOURSE_API_KEY @@ -2417,7 +2506,7 @@ def create_csv_from_dict(input_dict, title=None, filename=None): if os.path.isfile(download_file): os.remove(download_file) - with open(download_file, "w", newline='') as csvfile: + with open(download_file, "w", newline='', encoding='utf-8') as csvfile: if title: csvfile.write(title) csvfile.write("\n") @@ -2504,7 +2593,7 @@ def list(self, request): file_url = request.GET.get('file_url') if file_url and os.path.isfile(file_url): - with open(file_url) as csvfile: + with open(file_url, encoding='utf8') as csvfile: # return file and tidy up. response = HttpResponse(csvfile, content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=download.csv' @@ -2873,7 +2962,7 @@ def list(self, request): link = DownloadLinks.objects.filter(file_url=file_url) if (link and link[0].zip_file and os.path.isfile(link[0].file_url)): - logger.info('zip_file: {}'.format(link[0].zip_file)) + logger.info('zip_file: %s', link[0].zip_file) # return file and tidy up. file_name = os.path.basename(file_url) @@ -3081,90 +3170,82 @@ def create(self, request): content = {'Only authenticated users can transfer files'} return Response(content, status=status.HTTP_403_FORBIDDEN) - # Can't use this method if the squonk variables are not set! - if not settings.SQUONK2_DMAPI_URL: - content = {'SQUONK2_DMAPI_URL is not set'} + # Can't use this method if the Squonk2 agent is not configured + sq2a_rv = _SQ2A.configured() + if not sq2a_rv.success: + content = {f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'} return Response(content, status=status.HTTP_403_FORBIDDEN) + # Collect expected API parameters.... + access_id = request.data['access'] # The access ID/legacy Project record title target_id = request.data['target'] - target = Target.objects.get(id=target_id) snapshot_id = request.data['snapshot'] + session_project_id = request.data['session_project'] - if 'squonk_project' in request.data: - squonk_project = request.data['squonk_project'] - else: - content = { - 'message': 'A squonk project must be entered'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) + logger.info('+ user="%s" (id=%s)', user.username, user.id) + logger.info('+ access_id=%s', access_id) + logger.info('+ target_id=%s', target_id) + logger.info('+ snapshot_id=%s', snapshot_id) + logger.info('+ session_project_id=%s', session_project_id) + + target = Target.objects.get(id=target_id) + assert target + + # Check the user can use this Squonk2 facility. + # To do this we need to setup a couple of API parameter objects. + sq2a_common_params: CommonParams = CommonParams(user_id=user.id, + access_id=access_id, + session_id=session_project_id, + target_id=target_id) + sq2a_send_params: SendParams = SendParams(common=sq2a_common_params, + snapshot_id=snapshot_id) + sq2a_rv: Squonk2AgentRv = _SQ2A.can_send(sq2a_send_params) + if not sq2a_rv.success: + content = {f'You cannot do this ({sq2a_rv.msg})'} + return Response(content, status=status.HTTP_403_FORBIDDEN) + # Check the presense of the files expected to be transferred error, proteins, compounds = check_file_transfer(request) if error: return Response(error['message'], status=error['status']) - # If transfer has already happened find the latest - job_transfers = JobFileTransfer.objects.filter(snapshot=snapshot_id) - if job_transfers: - job_transfer = job_transfers.latest('id') - else: - job_transfer = None - if job_transfer and not job_transfer.target: - msg = f'JobTransfer record ({job_transfer.id})' \ - f' for snapshot {snapshot_id} has no target.' \ - ' Cannot continue' + # Create new file transfer job + logger.info('+ Calling ensure_project() to get the Squonk2 Project...') + + # This requires a Squonk2 Project (created by the Squonk2Agent). + # It may be an existing project, or it might be a new project. + common_params = CommonParams(user_id=user.id, + access_id=access_id, + target_id=target_id, + session_id=session_project_id) + sq2_rv = _SQ2A.ensure_project(common_params) + if not sq2_rv.success: + msg = f'Failed to get/create a Squonk2 Project' \ + f' for User "{user.username}", Access ID {access_id},' \ + f' Target ID {target_id}, and SessionProject ID {session_project_id}.' \ + f' Got "{sq2_rv.msg}".' \ + ' Cannot continue' content = {'message': msg} logger.error(msg) - return Response(content, - status=status.HTTP_404_NOT_FOUND) - - # The root (in the Squonk project) where files will be written. - # This is "understood" by the celery task (which uses this constant). - # e.g. 'fragalysis-files' - transfer_root = settings.SQUONK2_MEDIA_DIRECTORY - - logger.info('+ target_id=%s', target_id) - logger.info('+ snapshot_id=%s', snapshot_id) - logger.info('+ squonk_project=%s', squonk_project) - logger.info('+ transfer_root=%s', transfer_root) - - if job_transfer: - - # A pre-existing transfer... - transfer_target = job_transfer.target.title - if (job_transfer.transfer_status == 'PENDING' or - job_transfer.transfer_status == 'STARTED'): - - logger.info('+ Existing transfer_status=%s', job_transfer.transfer_status) - content = {'transfer_root': transfer_root, - 'transfer_target': transfer_target, - 'message': 'Files currently being transferred'} - return Response(content, - status=status.HTTP_208_ALREADY_REPORTED) - - if (target.upload_datetime and job_transfer.transfer_datetime) \ - and target.upload_datetime < job_transfer.transfer_datetime: - - # The target data has already been transferred for the snapshot. - logger.info('+ Existing transfer finished (transfer_status=%s)', - job_transfer.transfer_status) - content = {'transfer_root': transfer_root, - 'transfer_target': transfer_target, - 'message': 'Files already transferred for this job'} - return Response(content, - status=status.HTTP_200_OK) - - # Restart existing transfer - it must have failed or be outdated - job_transfer.user = request.user - - else: + return Response(content, status=status.HTTP_404_NOT_FOUND) - # Create new file transfer job - job_transfer = JobFileTransfer() - job_transfer.user = request.user - job_transfer.proteins = [p['code'] for p in proteins] - job_transfer.compounds = [c['name'] for c in compounds] - job_transfer.squonk_project = squonk_project - job_transfer.target = Target.objects.get(id=target_id) - job_transfer.snapshot = Snapshot.objects.get(id=snapshot_id) + # The Squonk2Project record is in the response msg + squonk2_project_uuid = sq2_rv.msg.uuid + squonk2_unit_name = sq2_rv.msg.unit.name + squonk2_unit_uuid = sq2_rv.msg.unit.uuid + logger.info('+ ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)', + squonk2_project_uuid, squonk2_unit_name, squonk2_unit_uuid) + + job_transfer = JobFileTransfer() + job_transfer.user = request.user + job_transfer.proteins = [p['code'] for p in proteins] + job_transfer.compounds = [c['name'] for c in compounds] + # We should use a foreign key, + # but to avoid migration issues with the existing code + # we continue to use the project UUID string field. + job_transfer.squonk_project = squonk2_project_uuid + job_transfer.target = Target.objects.get(id=target_id) + job_transfer.snapshot = Snapshot.objects.get(id=snapshot_id) # The 'transfer target' (a sub-directory of the transfer root) # For example the root might be 'fragalysis-files' @@ -3180,6 +3261,15 @@ def create(self, request): job_transfer.transfer_progress = None job_transfer.save() + # The root (in the Squonk project) where files will be written for this Job. + # Something like "fragalysis-files/hjyx" for new transfers, + # "fragalysis-files" for existing transfers + if job_transfer.sub_path: + transfer_root = os.path.join(settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path) + else: + transfer_root = settings.SQUONK2_MEDIA_DIRECTORY + logger.info('+ transfer_root=%s', transfer_root) + # Celery/Redis must be running. # This call checks and trys to start them if they're not. assert check_services() @@ -3235,8 +3325,9 @@ def list(self, request): return Response(content, status=status.HTTP_403_FORBIDDEN) # Can't use this method if the squonk variables are not set! - if not settings.SQUONK2_DMAPI_URL: - content = {'SQUONK2_DMAPI_URL is not set'} + sqa_rv = _SQ2A.configured() + if sqa_rv.success: + content = {f'The Squonk2 Agent is not configured ({sqa_rv.msg}'} return Response(content, status=status.HTTP_403_FORBIDDEN) job_collection = request.query_params.get('job_collection', None) @@ -3250,7 +3341,7 @@ def list(self, request): return Response(content) -class JobRequestView(viewsets.ModelViewSet): +class JobRequestView(APIView): """ Operational Django view to set up/retrieve information about tags relating to Session Projects @@ -3258,15 +3349,6 @@ class JobRequestView(viewsets.ModelViewSet): ------- url: api/job_request - queryset: - `viewer.models.JobRequest.objects.filter()` - filter fields: - - `viewer.models.JobRequest.snapshot` - ?snapshot=<int> - - `viewer.models.JobRequest.target` - ?target=<int> - - `viewer.models.JobRequest.user` - ?user=<int> - - `viewer.models.JobRequest.squonk_job_name` - ?squonk_job_name=<str> - - `viewer.models.JobRequest.squonk_project` - ?squonk_project=<str> - - `viewer.models.JobRequest.job_status` - ?job_status=<str> returns: JSON @@ -3293,30 +3375,78 @@ class JobRequestView(viewsets.ModelViewSet): """ - queryset = JobRequest.objects.filter() - filter_permissions = "target__project_id" - filterset_fields = ('id', 'snapshot', 'target', 'user', 'squonk_job_name', - 'squonk_project', 'job_status') + def get(self, request): + logger.info('+ JobRequest.get') + + user = self.request.user + if not user.is_authenticated: + content = {'Only authenticated users can access squonk jobs'} + return Response(content, status=status.HTTP_403_FORBIDDEN) - def get_serializer_class(self): - """Determine which serializer to use based on whether the request is a GET or a POST, PUT - or PATCH request + # Can't use this method if the Squonk2 agent is not configured + sq2a_rv = _SQ2A.configured() + if not sq2a_rv.success: + content = {f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'} + return Response(content, status=status.HTTP_403_FORBIDDEN) - Returns - ------- - Serializer (rest_framework.serializers.ModelSerializer): - - if GET: `viewer.serializers.JobRequestReadSerializer` - - if other: `viewer.serializers.JobRequestWriteSerializer - """ - if self.request.method in ['GET']: - # GET - return JobRequestReadSerializer - # (POST, PUT, PATCH) - return JobRequestWriteSerializer + # Iterate through each record, for JobRequests that are not 'finished' + # we call into Squonk to get an update. We then return the (possibly) updated + # records to the caller. - def create(self, request): - """Method to handle POST request - """ + results = [] + snapshot_id = request.query_params.get('snapshot', None) + + if snapshot_id: + logger.info('+ JobRequest.get snapshot_id=%s', snapshot_id) + job_requests = JobRequest.objects.filter(snapshot=int(snapshot_id)) + else: + logger.info('+ JobRequest.get snapshot_id=(unset)') + job_requests = JobRequest.objects.all() + + for jr in job_requests: + if not jr.job_has_finished(): + logger.info('+ JobRequest.get (id=%s) has not finished (job_status=%s)', + jr.id, jr.job_status) + + # Job's not finished, an opportunity to call into Squonk + # To get the current status. To do this we'll need + # the 'callback context' we supplied when launching the Job. + logger.info('+ JobRequest.get (id=%s, code=%s) getting update from Squonk...', + jr.id, jr.code) + sq2a_rv = _SQ2A.get_instance_execution_status(jr.code) + # If the job's now finished, update the record. + # If the call was successful we'll get None (not finished), + # 'LOST', 'SUCCESS' or 'FAILURE' + if not sq2a_rv.success: + logger.warning('+ JobRequest.get (id=%s, code=%s) check failed (%s)', + jr.id, jr.code, sq2a_rv.msg) + elif sq2a_rv.success and sq2a_rv.msg: + logger.info('+ JobRequest.get (id=%s, code=%s) new status is (%s)', + jr.id, jr.code, sq2a_rv.msg) + transition_time = str(datetime.utcnow()) + transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC) + jr.job_status = sq2a_rv.msg + jr.job_status_datetime = transition_time_utc + jr.job_finish_datetime = transition_time_utc + jr.save() + else: + logger.info('+ JobRequest.get (id=%s, code=%s) is (probably) still running', + jr.id, jr.code) + + serializer = JobRequestReadSerializer(jr) + results.append(serializer.data) + + num_results = len(results) + logger.info('+ JobRequest.get num_results=%s', num_results) + + # Simulate the original paged API response... + content = {'count': num_results, + 'next': None, + 'previous': None, + 'results': results} + return Response(content, status=status.HTTP_200_OK) + + def post(self, request): # Celery/Redis must be running. # This call checks and trys to start them if they're not. assert check_services() @@ -3328,9 +3458,37 @@ def create(self, request): content = {'Only authenticated users can run jobs'} return Response(content, status=status.HTTP_403_FORBIDDEN) - # Can't use this method if the squonk variables are not set! - if not settings.SQUONK2_DMAPI_URL: - content = {'SQUONK2_DMAPI_URL is not set'} + # Can't use this method if the Squonk2 agent is not configured + sq2a_rv = _SQ2A.configured() + if not sq2a_rv.success: + content = {f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'} + return Response(content, status=status.HTTP_403_FORBIDDEN) + + # Collect expected API parameters.... + target_id = request.data['target'] + snapshot_id = request.data['snapshot'] + session_project_id = request.data['session_project'] + access_id = request.data['access'] # The access ID/legacy Project record + + logger.info('+ user="%s" (id=%s)', user.username, user.id) + logger.info('+ access_id=%s', access_id) + logger.info('+ target_id=%s', target_id) + logger.info('+ snapshot_id=%s', snapshot_id) + logger.info('+ session_project_id=%s', session_project_id) + + # Check the user can use this Squonk2 facility. + # To do this we need to setup a couple of API parameter objects. + # We don't (at this point) care about the Job spec or callback URL. + sq2a_common_params: CommonParams = CommonParams(user_id=user.id, + access_id=access_id, + session_id=session_project_id, + target_id=target_id) + sq2a_run_job_params: RunJobParams = RunJobParams(common=sq2a_common_params, + job_spec=None, + callback_url=None) + sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params) + if not sq2a_rv.success: + content = {f'You cannot do this ({sq2a_rv.msg})'} return Response(content, status=status.HTTP_403_FORBIDDEN) try: @@ -3418,43 +3576,54 @@ def update(self, request, code=None): break if not status_changed: - logger.info('code=%s status=%s ignoring (no status change)', - code, status) + logger.info('+ JobCallBackView.update(code=%s) status=%s ignoring (no status change)', code, status) return HttpResponse(status=204) + # This is now a chance to safely set the squonk_url_ext using the instance ID + # present in the callback (if it's not already set). The instance is + # placed at the end of the string, and is expected to be found + # by process_compound_set_file() which loads the output file back + # into Fragalysis + if not jr.squonk_url_ext: + jr.squonk_url_ext = create_squonk_job_request_url(request.data['instance_id']) + logger.info("+ JobCallBackView.update(code=%s) jr.squonk_url_ext='%s'", code, jr.squonk_url_ext) + # Update the state transition time, # assuming UTC. transition_time = request.data.get('state_transition_time') if not transition_time: transition_time = str(datetime.utcnow()) - logger.warning("Callback is missing state_transition_time" - " (using '%s')", transition_time) + logger.warning("+ JobCallBackView.update(code=%s) callback is missing state_transition_time" + " (using '%s')", code, transition_time) transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC) jr.job_status_datetime = transition_time_utc - logger.info('code=%s status=%s transition_time=%s (new status)', + logger.info('+ JobCallBackView.update(code=%s) status=%s transition_time=%s (new status)', code, status, transition_time) # If the Job's start-time is not set, set it. if not jr.job_start_datetime: - logger.info('Setting job START datetime (%s)', transition_time) + logger.info('+ JobCallBackView.update(code=%s) setting job START datetime (%s)', + code, transition_time) jr.job_start_datetime = transition_time_utc # Set the Job's finish time (once) if it looks lie the Job's finished. # We can assume the Job's finished if the status is one of a number # of values... if not jr.job_finish_datetime and status in ('SUCCESS', 'FAILURE', 'REVOKED'): - logger.info('Setting job FINISH datetime (%s)', transition_time) + logger.info('+ JobCallBackView.update(code=%s) Setting job FINISH datetime (%s)', + code, transition_time) jr.job_finish_datetime = transition_time_utc - # Save - before going further. + # Save the JobRequest record before going further. jr.save() if status != 'SUCCESS': # Go no further unless SUCCESS return HttpResponse(status=204) - logger.info('Job finished (SUCCESS). Can we upload the results..?') + logger.info('+ JobCallBackView.update(code=%s) job finished (SUCCESS).' + ' Can we upload the results?', code) # SUCCESS ... automatic upload? # @@ -3479,21 +3648,21 @@ def update(self, request, code=None): job_output_path = '/' + os.path.dirname(job_output) job_output_filename = os.path.basename(job_output) - logging.info('job_output_path="%s"', job_output_path) - logging.info('job_output_filename="%s"', job_output_filename) + logging.info('+ JobCallBackView.update(code=%s) job_output_path="%s"', code, job_output_path) + logging.info('+ JobCallBackView.update(code=%s) job_output_filename="%s"', code, job_output_filename) # If it's not suitably named, leave expected_squonk_filename = 'merged.sdf' if job_output_filename != expected_squonk_filename: # Incorrectly named file - nothing to get/upload. - logger.info('SUCCESS but not uploading.' + logger.info('+ JobCallBackView.update(code=%s) SUCCESS but not uploading.' ' Expected "%s" as job_output_filename.' - ' Found "%s"', expected_squonk_filename, job_output_filename) + ' Found "%s"', code, expected_squonk_filename, job_output_filename) return HttpResponse(status=204) if jr.upload_status != 'PENDING': - logger.warning('SUCCESS but ignoring.' - ' upload_status=%s (already uploading?)', jr.upload_status) + logger.warning('+ JobCallBackView.update(code=%s) SUCCESS but ignoring.' + ' upload_status=%s (already uploading?)', code, jr.upload_status) return HttpResponse(status=204) # Change of status and SUCCESS @@ -3516,7 +3685,119 @@ def update(self, request, code=None): erase_compound_set_job_material.s(job_request_id=jr.id) ).apply_async() - logger.info('Started process_job_file_upload(%s) task_upload=%s', - jr.id, task_upload) + logger.info('+ JobCallBackView.update(code=%s)' + ' started process_job_file_upload(%s) task_upload=%s', + code, jr.id, task_upload) return HttpResponse(status=204) + +class JobAccessView(APIView): + """Django view that calls Squonk to allow a user (who is able to see a Job) + the ability to access that Job in Squonk. To be successful the user + must have access to the corresponding Fragalysis Project. This can be called by + the Job 'owner', who always has access. + + Methods + ------- + allowed requests: + - GET: Get job access + + url: + api/job_access + get params: + - job_request_id: The identity of the JobRequest (the Job) the user needs access to + + Returns: A structure with 'accessible' set to True on success. On failure + an 'error' string contains a reason + + example input for get + + .. code-block:: + + /api/job_access/?job_request_id=17 + """ + def get(self, request): + """Method to handle GET request + """ + query_params = request.query_params + logger.info('+ JobAccessView/GET %s', json.dumps(query_params)) + + err_response = {'accessible': False} + ok_response = {'accessible': True, 'error': ''} + + # Only authenticated users can have squonk jobs + user = self.request.user + if not user.is_authenticated: + content = {'accessible': False, + 'error': 'Only authenticated users can access Squonk Jobs'} + return Response(content, status=status.HTTP_403_FORBIDDEN) + + # Can't use this method if the squonk variables are not set! + sqa_rv = _SQ2A.configured() + if not sqa_rv.success: + err_response['error'] = f'Squonk is not available ({sqa_rv.msg})' + return Response(err_response, status=status.HTTP_403_FORBIDDEN) + + # Get the JobRequest Record form the supplied record ID + jr_id_str = request.query_params.get('job_request_id', None) + if not jr_id_str or not jr_id_str.isdigit(): + err_response['error'] = f'The JobRequest ID ({jr_id_str}) is not valid' + return Response(err_response, status=status.HTTP_400_BAD_REQUEST) + jr_id = int(jr_id_str) + if jr_id < 1: + err_response['error'] = f'The JobRequest ID ({jr_id}) cannot be less than 1' + return Response(err_response, status=status.HTTP_400_BAD_REQUEST) + + jr_list = JobRequest.objects.filter(id=jr_id) + if len(jr_list) == 0: + err_response['error'] = f'The JobRequest does not exist' + return Response(err_response, status=status.HTTP_400_BAD_REQUEST) + jr = jr_list[0] + + # JobRequest must have a Squonk Project value + if not jr.squonk_project: + err_response['error'] = f'The JobRequest ({jr_id}) has no Squonk Project value' + return Response(err_response, status=status.HTTP_403_FORBIDDEN) + + # User must have access to the Job's Project. + # If the user is not the owner of the Job, and there is a Project, + # we then check the user has access to the gievn access ID. + # + # If the Job has no Project (Jobs created before this chnage will not have a Project) + # or the user is the owner of the Job we skip this check. + if user.id != jr.user.id: + logger.info('+ JobAccessView/GET Checking access to JobRequest %s for "%s" (%s)', + jr_id, user.username, jr.squonk_project) + if not jr.project or not jr.project.title: + logger.warning('+ JobAccessView/GET No Fragalysis Project (or title)' + ' for JobRequest %s - granting access', + jr_id) + else: + # The project is the Job's access ID. + # To check access we need this and the User's ID + access_id = jr.project.id + access_string = jr.project.title + sq2a_common_params: CommonParams = CommonParams(user_id=user.id, + access_id=access_id, + session_id=None, + target_id=None) + sq2a_run_job_params: RunJobParams = RunJobParams(common=sq2a_common_params, + job_spec=None, + callback_url=None) + sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params) + if not sq2a_rv.success: + err_response['error'] = f'Access to the Job for {access_id} ({access_string}) is denied. {sq2a_rv.msg}' + return Response(err_response, status=status.HTTP_403_FORBIDDEN) + + # All checks passed ... grant access for this user + sq2a_access_params: AccessParams = AccessParams(username=user.username, + project_uuid=jr.squonk_project) + sqa_rv = _SQ2A.grant_access(sq2a_access_params) + if not sqa_rv.success: + err_response['error'] = f'Squonk failed to grant access ({sqa_rv.msg})' + logger.warning('+ JobAccessView/GET error=%s', content['error']) + return Response(err_response, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + logger.info('+ JobAccessView/GET Success for %s/"%s" on %s', + jr_id, user.username, jr.squonk_project) + return Response(ok_response)