From 54bbd4af1821e24b873f105699b369f38125e007 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 26 Oct 2022 09:58:21 +0100 Subject: [PATCH 001/112] Early work on squonk2 "agent" --- README.md | 4 +- docker-compose.yml | 23 +- requirements.txt | 2 +- viewer/migrations/0025_squonk2org.py | 23 ++ viewer/models.py | 71 +++- viewer/squonk2_agent.py | 462 +++++++++++++++++++++++++++ 6 files changed, 575 insertions(+), 10 deletions(-) create mode 100644 viewer/migrations/0025_squonk2org.py create mode 100644 viewer/squonk2_agent.py diff --git a/README.md b/README.md index ccc9d9d6..4668ca39 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Now the project's rules will run on every commit. Start `Fragalysis stack` (All infrastructure - databases + populating data) ``` -docker-compose -f docker-compose.dev.yml up -d +docker-compose up -d ``` `Please wait, it takes a minute until all containers are fully started.` @@ -133,7 +133,7 @@ Test if we are running at [http://localhost:8080](http://localhost:8080) If needed stop containers ``` -docker-compose -f docker-compose.dev.yml down +docker-compose down ``` diff --git a/docker-compose.yml b/docker-compose.yml index 0c4c284f..e3fc1cb2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,14 +54,25 @@ services: POSTGRESQL_PASSWORD: fragalysis POSTGRESQL_HOST: database POSTGRESQL_PORT: 5432 + # Celery tasks run synchronously? + CELERY_TASK_ALWAYS_EAGER: 'True' # Default Debug to true for local development - DEBUG_FRAGALYSIS: 'True' - DISABLE_LOGGING_FRAMEWORK: 'No' OIDC_RP_CLIENT_SECRET: 'c6245428-04c7-466f-9c4f-58c340e981c2' - SQUONK2_UI_URL: https://data-manager-ui.xchem-dev.diamond.ac.uk/data-manager-ui - SQUONK2_DMAPI_URL: https://data-manager.xchem-dev.diamond.ac.uk/data-manager-api - # Celery tasks run synchronously? -# CELERY_TASK_ALWAYS_EAGER: 'True' + SQUONK2_VERIFY_CERTIFICATES: 'No' + SQUONK2_UNIT_BILLING_DAY: 3 + SQUONK2_PRODUCT_FLAVOUR: Bronze + SQUONK2_SLUG: fs-local + # The following are normally populated via a local '.env' file + # that is not part of the repo, it is a file you create + # and is used by docker-compose if it finds it. + OIDC_KEYCLOAK_REALM: ${OIDC_KEYCLOAK_REALM} + OIDC_AS_CLIENT_ID: ${OIDC_AS_CLIENT_ID} + SQUONK2_ORG_OWNER: ${SQUONK2_ORG_OWNER} + SQUONK2_ORG_OWNER_PASSWORD: ${SQUONK2_ORG_OWNER_PASSWORD} + SQUONK2_ORG_UUID: ${SQUONK2_ORG_UUID} + SQUONK2_UI_URL: ${SQUONK2_UI_URL} + SQUONK2_DMAPI_URL: ${SQUONK2_DMAPI_URL} + SQUONK2_ASAPI_URL: ${SQUONK2_ASAPI_URL} ports: - "8080:80" depends_on: diff --git a/requirements.txt b/requirements.txt index 8d0249f2..2809b491 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.3.0 +im-squonk2-client==1.12.0 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 diff --git a/viewer/migrations/0025_squonk2org.py b/viewer/migrations/0025_squonk2org.py new file mode 100644 index 00000000..209f55d8 --- /dev/null +++ b/viewer/migrations/0025_squonk2org.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.14 on 2022-10-19 09:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0024_add_job_request_start_and_finish_times'), + ] + + operations = [ + migrations.CreateModel( + name='Squonk2Org', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.TextField(max_length=40)), + ('name', models.TextField(max_length=80)), + ('as_url', models.URLField()), + ('as_version', models.TextField()), + ], + ), + ] diff --git a/viewer/models.py b/viewer/models.py index ba4dfdb7..24d7ff96 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -1201,6 +1201,75 @@ class JobRequest(models.Model): class Meta: db_table = 'viewer_jobrequest' -# End of Squonk Job Tables +class Squonk2Org(models.Model): + """Django model to store Squonk2 Organisations (UUIDs) and the Account Servers + they belong to. Managed by the Squonk2Agent class. + + Parameters + ---------- + uuid: TextField (40) + A Squonk2 Account Server (AS) Organisation UUID. A fixed length string + consisting of 'org-' followed by a uuid4 value, + e.g. 'org-54260047-183b-42e8-9658-385a1e1bd236' + name: TextField (80) + The name of the Squonk2 Organisation UUID (obtained form the AS). + as_url: URLField (200) + The URL of the Squonk2 Account Server that owns the organisation. + e.g. 'https://example.com/account-server-api' + as_version: TextField + The version of the AS that was first seen to own the Organisation + """ + uuid = models.TextField(max_length=40, null=False) + name = models.TextField(max_length=80, null=False) + as_url = models.URLField(null=False) + as_version = models.TextField(null=False) + +class Squonk2Unit(models.Model): + """Django model to store Squonk2 Unit (UUIDs). Managed by the Squonk2Agent class. + + Parameters + ---------- + uuid: TextField (41) + A Squonk2 Account Server (AS) Unit UUID. A fixed length string + consisting of 'unit-' followed by a uuid4 value, + e.g. 'unit-54260047-183b-42e8-9658-385a1e1bd236' + name: TextField (80) + The name of the Squonk2 Unit UUID (obtained form the AS). + project: ForeignKey + A Foreign Key to the Project (Proposal) the Unit belongs to. + organisation: ForeignKey + A Foreign Key to the Organisation the Unit belongs to. + """ + uuid = models.TextField(max_length=41, null=False) + name = models.TextField(max_length=80, null=False) + + project = models.ForeignKey(Project, null=False, on_delete=models.CASCADE) + organisation = models.ForeignKey(Squonk2Org, null=False, on_delete=models.CASCADE) + +class Squonk2Project(models.Model): + """Django model to store Squonk2 Project (UUIDs). Managed by the Squonk2Agent class. + + Parameters + ---------- + uuid: TextField (44) + A Squonk2 Data Manager (DM) Project UUID. A fixed length string + consisting of 'project-' followed by a uuid4 value, + e.g. 'project-54260047-183b-42e8-9658-385a1e1bd236' + name: TextField (80) + The name of the Squonk2 Unit UUID (obtained form the AS). + product_uuid: TextField (44) + A Squonk2 Account Server (AS) Product UUID. A fixed length string + consisting of 'product-' followed by a uuid4 value, + e.g. 'product-54260047-183b-42e8-9658-385a1e1bd236' + """ + uuid = models.TextField(max_length=44, null=False) + name = models.TextField(max_length=80, null=False) + product_uuid = models.TextField(max_length=44, null=False) + + unit = models.ForeignKey(Squonk2Unit, null=False, on_delete=models.CASCADE) + user = models.ForeignKey(User, null=False, on_delete=models.CASCADE) + target = models.ForeignKey(Target, null=False, on_delete=models.CASCADE) + +# End of Squonk Job Tables diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py new file mode 100644 index 00000000..19b2e3bb --- /dev/null +++ b/viewer/squonk2_agent.py @@ -0,0 +1,462 @@ +"""An agent for the Squonk2 (Data Manger and Account Server) API. +This module 'simplifies' the use of the Squonk2 Python client package. +""" + +# Refer to the accompanying low-level-design document: - +# https://docs.google.com/document/d/1lFpN29dK1luz80lwSGi0Rnj1Rqula2_CTRuyWDUBu14 + +from collections import namedtuple +import logging +import os +from typing import List, Optional, Tuple +from urllib.parse import ParseResult, urlparse +from urllib3.exceptions import InsecureRequestWarning +from urllib3 import disable_warnings + +from django.core.exceptions import ObjectDoesNotExist +from squonk2.auth import Auth +from squonk2.as_api import AsApi, AsApiRv +import requests +from requests import Response +from wrapt import synchronized + +from viewer.models import Squonk2Project, Squonk2Org, Squonk2Unit + +_LOGGER: logging.Logger = logging.getLogger(__name__) + +# Response value for the agent methods +Squonk2AgentRv: namedtuple = namedtuple('Squonk2AgentRv', ['success', 'msg']) +SuccessRv: Squonk2AgentRv = Squonk2AgentRv(succes=True, msg=None) + +# Named tuples are used to pass parameters to the agent methods. +# RunJob, used in run_job() +RunJob: namedtuple = namedtuple("RunJob", ["access_token", + "proposal", + "user_id", + "target_id", + "job_spec", + "callback_url"]) + +# Send, used in send() +Send: namedtuple = namedtuple("Send", ["access_token", + "proposal", + "user_id", + "target_id", + "snapshot_id"]) + + +_SUPPORTED_PRODUCT_FLAVOURS: List[str] = ["BRONZE", "SILVER", "GOLD"] + +_MAX_SLUG_LENGTH: int = 10 + + +class Squonk2Agent: + """Helper class that simplifies access to the Squonk2 Python client. + Users shouldn't instantiate the class directly, instead they should + get access to the class singleton via a call to 'get_squonk2_agent()'. + + The class methods protect the caller from using them unless a) the class has + sufficient configuration and b) the Squonk2 services are 'alive'. + """ + + def __init__(self): + """Initialise the instance, loading from the environment. + """ + + # Primary configuration of the module is via the container environment. + # We need to recognise that some or all of these may not be defined. + # All run-time config that's required is given a __CFG prefix to + # simplify checkign whether all that's required has been defined. + self.__CFG_SQUONK2_ASAPI_URL: Optional[str] =\ + os.environ.get('SQUONK2_ASAPI_URL') + self.__CFG_SQUONK2_DMAPI_URL: Optional[str] =\ + os.environ.get('SQUONK2_DMAPI_URL') + self.__CFG_SQUONK2_UI_URL: Optional[str] =\ + os.environ.get('SQUONK2_UI_URL') + self.__CFG_SQUONK2_ORG_UUID: Optional[str] =\ + os.environ.get('SQUONK2_ORG_UUID') + self.__CFG_SQUONK2_UNIT_BILLING_DAY: Optional[str] =\ + os.environ.get('SQUONK2_UNIT_BILLING_DAY') + self.__CFG_SQUONK2_PRODUCT_FLAVOUR: Optional[str] =\ + os.environ.get('SQUONK2_PRODUCT_FLAVOUR') + self.__CFG_SQUONK2_SLUG: Optional[str] =\ + os.environ.get('SQUONK2_SLUG') + self.__CFG_SQUONK2_ORG_OWNER: Optional[str] =\ + os.environ.get('SQUONK2_ORG_OWNER') + self.__CFG_SQUONK2_ORG_OWNER_PASSWORD: Optional[str] =\ + os.environ.get('SQUONK2_ORG_OWNER_PASSWORD') + self.__CFG_OIDC_AS_CLIENT_ID: Optional[str] = \ + os.environ.get('OIDC_AS_CLIENT_ID') + self.__CFG_OIDC_KEYCLOAK_REALM: Optional[str] = \ + os.environ.get('OIDC_KEYCLOAK_REALM') + + # Optional config (no '__CFG_' prefix) + self.__FALLBACK_PROPOSAL_ID: Optional[str] =\ + os.environ.get('FALLBACK_PROPOSAL_ID') + self.__FORCE_FALLBACK_PROPOSAL_ID: Optional[str] =\ + os.environ.get('FORCE_FALLBACK_PROPOSAL_ID') + self.__SQUONK2_VERIFY_CERTIFICATES: Optional[str] = \ + os.environ.get('SQUONK2_VERIFY_CERTIFICATES') + + # The integer billing day, valid if greater than zero + self.__unit_billing_day: int = 0 + # The product tier, valid if set + self.__product_flavour: str = '' + # True if configured... + self.__configuration_checked: bool = False + self.__configured: bool = False + # OIDC hostname and realm. + # Extracted during configuration check from the OIDC variable + self.__oidc_hostname: str = '' + self.__oidc_realm: str = '' + # Ignore cert errors? (no) + self.__verify_certificates: bool = True + + # Set when pre-flight checks have passed. + # When they've been done we can safely (?) continue to use the + # Squonk2 Python client. + self.__pre_flight_check_status: bool = False + # The record ID of the Squonk2Org for this deployment. + # Set on successful 'pre-flight-check' + self.__org_record: Optional[Squonk2Org] = None + + self.__owner_token: str = '' + + def _get_org_owner_token(self) -> Optional[str]: + """Gets an access token for the Squonk2 organisation owner. + This sets the __keycloak_hostname member and also returns the token. + """ + assert self.__keycloak_hostname + self.__owner_token = Auth.get_access_token( + keycloak_url="https://" + self.__keycloak_hostname + "/auth", + keycloak_realm=self.__keycloak_realm, + keycloak_client_id=self.__CFG_OIDC_AS_CLIENT_ID, + username=self.__CFG_SQUONK2_ORG_OWNER, + password=self.__CFG_SQUONK2_ORG_OWNER_PASSWORD, + ) + if not self.__owner_token: + _LOGGER.warning('Failed to get access token for Squonk2 org owner') + return None + # OK if we get here + return self.__owner_token + + def _pre_flight_checks(self) -> Squonk2AgentRv: + """Execute pre-flight checks, + can be called multiple times, it acts only once. + """ + # Been here before, and successful? + # If not try the pre-flight check again. + if self.__pre_flight_check_status: + return Squonk2AgentRv(success=True, msg=None) + + # If a Squonk2Org record exists its UUID cannot have changed. + # We cannot change the organisation once deployed. The corresponding Units, + # Products and Projects are organisation-specific. The Squonk2Org table + # records the organisation ID and the Account Server URL where the ID + # is valid. Neither of these values can change once deployed. + + squonk2_org: Optional[Squonk2Org] = Squonk2Org.objects.all().first() + if squonk2_org and squonk2_org.uuid != self.__CFG_SQUONK2_ORG_UUID: + msg: str = f'Configured Squonk2 Organisation ({self.__CFG_SQUONK2_ORG_UUID})'\ + f' does not match pre-existing record ({squonk2_org.uuid})' + return Squonk2AgentRv(success=False, msg=msg) + + # OK, so the ORG UUID has not changed. + # Is it known to the configured AS? + if not self._get_org_owner_token(): + msg = 'Failed to get AS token for organisation owner' + return Squonk2AgentRv(success=False, msg=msg) + + # Get the ORG from the AS API. + # If it knows the org the response will be successful, + # and we'll also have the Org's name. + as_rv = AsApi.get_organisation(self.__owner_token, + org_id=self.__CFG_SQUONK2_ORG_UUID) + if not as_rv.success: + msg = 'Failed to get Organisation from Account Server' + print(msg) + return quonk2AgentRv(success=False, msg=msg) + + # The org is known to the AS. + # Get the AS API version (for reference) + as_rv: AsApiRv = AsApi.get_version() + if not as_rv.success: + msg = 'Failed to get version from Account Server' + print(msg) + return quonk2AgentRv(success=False, msg=msg) + as_version: str = as_rv.msg['version'] + + # If there's no Squonk2Org record, create one, + # recording the ORG ID and the AS we used to verify it exists. + if not squonk2_org: + squonk2_org = Squonk2Org(uuid=self.__CFG_SQUONK2_ORG_UUID, + name=as_rv.msg['name'], + as_url=self.__CFG_SQUONK2_ASAPI_URL, + as_version=as_version) + squonk2_org.save() + + # Keep the record ID for future use. + self.__org_record = squonk2_org + + # Organisation is known to AS, and it hasn't changed. + return SuccessRv + + def _ensure_unit(self, proposal: str) -> Squonk2AgentRv: + """Gets or creates a Squonk2 Unit. + + On success the returned message is used to carry the Squonk2 project UUID. + """ + assert proposal + assert self.__org_record + + unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(proposal=proposal).first() + if not unit: + unit_name: styr = f'Fragalysis {self.__CFG_SQUONK2_SLUG} {proposal}' + rv: AsApiRv = AsApi.create_unit(unit_name=unit_name, + org_id=self.__org_record.uuid, + billing_day=self.__unit_billing_day) + if not rv.success: + msg: str = rv.msg['error'] + return Squonk2AgentRv(success=False, msg=msg) + unit_uuid: str = rv.msg['id'] + unit: Squonk2Unit = Squonk2Unit(uuid=unit_uuid, + name=unit_name, + proposal=proposal, + organisation=self.__org_record.id) + unit.save() + + return Squonk2AgentRv(success=True, msg=unit.uuid) + + def _ensure_project(self, + user_id: int, + target_id: int, + proposal: str) -> Squonk2AgentRv: + """Gets or creates a Squonk2 Project, used as the destination of files + and job executions. Each project requires an AS Product + (tied to the User and Target) and Unit (tied to the proposal). + + The proposal is expected to be valid for a given user, this method does not + check whether the user/proposal combination - it assumes that what's been + given has been checked. + + On success the returned message is used to carry the Squonk2 project UUID. + """ + assert user_id + assert user_id > 0 + assert target_id + assert target_id > 0 + assert proposal + + # A Squonk2Unit must exist for the Proposal, and there must be + # a Squonk2Project record for the user/target combination. + # If not it is created. + rv: Squonk2AgentRv = self._ensure_unit(proposal) + if not rv.success: + return rv + unit_uuid: str = rv.msg + + # A Squonk2Project record must exist for the unit/user/target combination. + # If not it is created. + project: Optional[Squonk2Project] =\ + Squonk2Project.objects.filter(user__id=user_id, + target__id=target_id, + unit__uuid=unit_uuid)\ + .first() + if not project: + # Need to call upon Squonk2 to create a 'Product' and a 'Project'. + # The Product is created by the organisation 'owner' + # the 'Project' is created on behalf of the 'user'. + # + # We create a corresponding Squonk2Project when both have been successful. + rv = self._create_project() + if not rv.success: + return rv + project = Squonk2Project(uuid=project_uuid, + name=project_name, + product_uuid=product_uuid, + unit=unit_id, + user=user_id, + target=target_id) + project.save() + + return Squonk2AgentRv(success=True, msg=project.uuid) + + @property + def configured(self) -> Squonk2AgentRv: + """Returns True if the module appears to be configured, + i.e. all the environment variables appear to be set. + """ + # To prevent repeating the checks, all of which are based on + # static (environment) variables, if we've been here before + # just return our previous result. + if self.__configuration_checked: + return self.__configured, None + + self.__configuration_checked = True + for name, value in self.__dict__.items(): + # All required configuration has a class '__CFG' prefix + if name.startswith('_Squonk2Agent__CFG_'): + if value is None: + cfg_name: str = name.split('_Squonk2Agent__CFG_')[1] + msg: str = f'{cfg_name} is not set' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # If we get here all the required configuration variables are set + + # Is the slug too long? + # Limited to 10 characters + if len(self.__CFG_SQUONK2_SLUG) > _MAX_SLUG_LENGTH: + msg: str = f'Slug is longer than {_MAX_SLUG_LENGTH} characters'\ + f' ({self.__CFG_SQUONK2_SLUG})' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Extract hostname and realm from the legacy variable + # i.e. we need 'example.com' and 'xchem' + # from 'https://example.com/auth/realms/xchem' + url: ParseResult = urlparse(self.__CFG_OIDC_KEYCLOAK_REALM) + self.__keycloak_hostname = url.hostname + self.__keycloak_realm = os.path.split(url.path)[1] + + # Can we translate the billing day to an integer? + if not self.__CFG_SQUONK2_UNIT_BILLING_DAY.isdigit(): + msg = 'SQUONK2_UNIT_BILLING_DAY is set'\ + ' but the value is not a number'\ + f' ({ self.__CFG_SQUONK2_UNIT_BILLING_DAY})' + _LOGGER.error(msg) + return False, msg + self.__unit_billing_day = int(self.__CFG_SQUONK2_UNIT_BILLING_DAY) + if self.__unit_billing_day < 1: + msg = 'SQUONK2_UNIT_BILLING_DAY cannot be less than 1' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Product tier to upper-case + if not self.__CFG_SQUONK2_PRODUCT_FLAVOUR in _SUPPORTED_PRODUCT_FLAVOURS: + msg = 'SQUONK2_PRODUCT_FLAVOUR is not supported' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + self.__product_flavour = self.__CFG_SQUONK2_PRODUCT_FLAVOUR.upper() + + # Don't verify Squonk2 SSL certificates? + if self.__SQUONK2_VERIFY_CERTIFICATES and self.__SQUONK2_VERIFY_CERTIFICATES.lower() == 'no': + self.__verify_certificates = False + disable_warnings(InsecureRequestWarning) + + # OK - it all looks good. + # Mark as 'configured' + self.__configured = True + + return SuccessRv + + @property + def ping(self) -> Squonk2AgentRv: + """Returns True if all the Squonk2 installations + referred to by the URLs respond. + + We also validate that the organisation supplied is known to the Account Server + by calling on '_pre_flight_checks()'. If the org is known a Squonk2Org record + is created, if not the ping fails. + """ + if not self.configured: + msg: str = 'Not configured' + _LOGGER.debug(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Check the UI, DM and AS... + + resp: Optional[Response] = None + url: str = self.__CFG_SQUONK2_UI_URL + try: + resp: Response = requests.head(url, verify=self.__verify_certificates) + except: + _LOGGER.warning('Exception checking UI at %s', url) + if resp is None or resp.status_code != 200: + msg = f'UI is not responding from {url}' + print(msg) + _LOGGER.debug(msg) + return Squonk2AgentRv(success=False, msg=msg) + + resp = None + url = f'{self.__CFG_SQUONK2_DMAPI_URL}/api' + try: + resp = requests.head(url, verify=self.__verify_certificates) + except: + _LOGGER.warning('Exception checking DM at %s', url) + if resp is None or resp.status_code != 308: + msg = f'Data Manager is not responding from {url}' + print(msg) + _LOGGER.debug(msg) + return Squonk2AgentRv(success=False, msg=msg) + + resp = None + url = f'{self.__CFG_SQUONK2_ASAPI_URL}/api' + try: + resp = requests.head(url, verify=self.__verify_certificates) + except: + _LOGGER.warning('Exception checking AS at %s', url) + if resp is None or resp.status_code != 308: + msg = f'Account Manager is not responding from {url}' + print(msg) + _LOGGER.debug(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # OK so far. + # Is the configured organisation known to the AS (and has it changed?) + status, msg = self._pre_flight_checks() + if not status: + msg = f'Failed pre-flight checks ({msg})' + return Squonk2AgentRv(success=False, msg=msg) + + # Everything's responding if we get here... + return SuccessRv + + def run_job(self, params: RunJob) -> Squonk2AgentRv: + """Executes a Job on a Squonk2 installation. + """ + assert params + assert isinstance(params, RunJob) + + # Protect against lack of config or connection/setup issues... + if not self.ping: + msg: str = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk alive?' + return Squonk2AgentRv(success=False, msg=msg) + + return SuccessRv + + def send(self, params: Send) -> Squonk2AgentRv: + """A blocking method that takes care of send a set of files to + the configured Squonk2 installation. + """ + assert params + assert isinstance(params, Send) + + # Protect against lack of config or connection/setup issues... + if not self.ping: + msg: str = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk alive?' + return Squonk2AgentRv(success=False, msg=msg) + + return SuccessRv + + +# The global (singleton). +# This acts as out sole singleton, +# created and returned from `get_squonk2_agent()' +_SQUONK2_AGENT: Optional[Squonk2Agent] = None + + +def get_squonk2_agent() -> Squonk2Agent: + """Returns a 'singleton'. + """ + global _SQUONK2_AGENT # pylint: disable=global-statement + + if _SQUONK2_AGENT: + return _SQUONK2_AGENT + _LOGGER.debug("Creating new Squonk2Agent...") + _SQUONK2_AGENT = Squonk2Agent() + _LOGGER.debug("Created") + + return _SQUONK2_AGENT From b4e03d5b53e6ece87a45faa18672161f9b5631e9 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 26 Oct 2022 13:52:00 +0100 Subject: [PATCH 002/112] Fix target upload (duplicate target) - Fixed exception if protein exists - Initial work on relaxing Protein constraints --- fix-target-file.py | 0 ...025_protein_target_id_unique_constraint.py | 17 +++++++ viewer/models.py | 2 +- viewer/target_set_upload.py | 51 ++++++++++++------- 4 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 fix-target-file.py create mode 100644 viewer/migrations/0025_protein_target_id_unique_constraint.py diff --git a/fix-target-file.py b/fix-target-file.py new file mode 100644 index 00000000..e69de29b diff --git a/viewer/migrations/0025_protein_target_id_unique_constraint.py b/viewer/migrations/0025_protein_target_id_unique_constraint.py new file mode 100644 index 00000000..1bffaa68 --- /dev/null +++ b/viewer/migrations/0025_protein_target_id_unique_constraint.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1.14 on 2022-10-26 09:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0024_add_job_request_start_and_finish_times'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='protein', + unique_together={('code', 'target_id', 'prot_type')}, + ), + ] diff --git a/viewer/models.py b/viewer/models.py index ba4dfdb7..1aa155f7 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -165,7 +165,7 @@ class Protein(models.Model): has_eds = models.NullBooleanField() class Meta: - unique_together = ("code", "prot_type") + unique_together = ("code", "target_id", "prot_type") class Compound(models.Model): diff --git a/viewer/target_set_upload.py b/viewer/target_set_upload.py index d48c87a4..342f5981 100644 --- a/viewer/target_set_upload.py +++ b/viewer/target_set_upload.py @@ -172,7 +172,7 @@ def add_prot(code, target, xtal_path, xtal, input_dict): """Add a protein with a PDB, code and :param code: the unique code for this file - :param target: the target to be linkede to + :param target: the target to be linked to :param xtal_path: the path to the crystal directory :param xtal: name of the xtal(?) :param input_dict: dictionary of files in crystal directory from load_dir @@ -183,13 +183,13 @@ def add_prot(code, target, xtal_path, xtal, input_dict): # code is normally the xtal directory in the aligned folder, but this may have been modified to have # an alternate name added to it - in the form 'directory:alternate_name'. code_first_part = code.split(":")[0] - proteins = Protein.objects.filter(code__contains=code_first_part) + proteins = Protein.objects.filter(code__contains=code_first_part, target_id=target) if proteins.exists(): new_prot = proteins.first() - logger.debug("Protein exists='%s'", new_prot[1]) + logger.debug("Pre-existing Protein (%s)", new_prot) else: new_prot = Protein.objects.get_or_create(code=code, target_id=target) - logger.debug("Protein created new_prot='%s'", new_prot[1]) + logger.debug("New Protein (code='%s' target_id='%s')", code, target) new_prot = new_prot[0] new_prot.apo_holo = True @@ -1178,20 +1178,37 @@ def validate_target(new_data_folder, target_name, proposal_ref): # Check if there is any data to process target_path = os.path.join(new_data_folder, target_name) - if not os.path.isdir(target_path): - validate_dict = add_tset_warning(validate_dict, 'Folder', f'No folder matching target name in extracted zip file {new_data_folder}, {target_name}', 0) - - aligned_path = os.path.join(target_path, 'aligned') - - if not os.path.isdir(aligned_path): - validate_dict = add_tset_warning(validate_dict, 'Folder', 'No aligned folder present in target name folder', 0) + # Assume success... + validated = True - # Check if there is a metadata.csv file to process - metadata_file = os.path.join(aligned_path, 'metadata.csv') - if os.path.isfile(metadata_file): - validated, validate_dict = check_metadata(metadata_file, validate_dict) - else: - validate_dict = add_tset_warning(validate_dict, 'File', 'No metedata.csv file present in the aligned folder', 0) + # A target directory must exist + if not os.path.isdir(target_path): + validate_dict = add_tset_warning(validate_dict, 'Folder', + 'Folder does not match target name.' + f' Expected "{target_name}".' + f' Is the upload called "{target_name}.zip"?', 0) + # No point in checking anything else if this check fails validated = False + if validated: + # An 'aligned' directory must exist + aligned_path = os.path.join(target_path, 'aligned') + if not os.path.isdir(aligned_path): + validate_dict = add_tset_warning(validate_dict, 'Folder', + 'No aligned folder present.' + f' Expected "{target_name}/{aligned_path}"', 0) + # No point in checking anything else if this check fails + ok_so_far = False + + if validated: + # A metadata.csv file must exist + metadata_file = os.path.join(aligned_path, 'metadata.csv') + if os.path.isfile(metadata_file): + validated, validate_dict = check_metadata(metadata_file, validate_dict) + else: + validate_dict = add_tset_warning(validate_dict, 'File', + 'No metedata file present.' + f' Expected "{target_name}/{aligned_path}/{metadata_file}"', 0) + validated = False + return validated, validate_dict From 612383b94475995af0d9d4487f57944f49040a76 Mon Sep 17 00:00:00 2001 From: Tim Dudgeon Date: Wed, 30 Nov 2022 14:08:43 +0000 Subject: [PATCH 003/112] docs on vecrtors --- docs/source/index.rst | 1 + docs/source/vector-mols.png | Bin 0 -> 38918 bytes docs/source/vectors.rst | 61 ++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 docs/source/vector-mols.png create mode 100644 docs/source/vectors.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index dad34f66..8dbbc939 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,6 +25,7 @@ Contents computational_data/tasks API/media API/download + vectors Indices and tables diff --git a/docs/source/vector-mols.png b/docs/source/vector-mols.png new file mode 100644 index 0000000000000000000000000000000000000000..1b8fe6b345ef7fcdebbfe4e964928ef0a50ad0c3 GIT binary patch literal 38918 zcmdq}WmJ^k8$Sw<9VjSLQX(xN3<}a9N;gQSAR#dzEsdy1Bi$j*P|`A#inM^_NPne9 z7(y9B(sPZ!|2glU=f$(ud0Z?RapsP__r0%AU4&|=$zQ%mdl7*^Tvk+&(Lx|jO(PH| z?a!ZupB!=hGK6oZT^}fFpNBub=Pg6v-`Cw__1zvhS-W|fyI3J?9Gy^Bw_TsOSXnu` zK6P^2IMphNKrkT`Wu&#Ar>{=?d|1`?`t#fK6xRdU6Wkj49FHg#Eo#FaEYf>u#k(-_ zu4p!BKPGRWh^IOeFRu~%yf~z%k8aoU6&2c;E9Cwejy4e{UCt1Vr)BJP8pX^mSAKAW z>>WAl-@F^wu$E$MJ+Mm<$GwoK-p%wJU^Vcc3yMO-?+B5b z|8GAC#avw0VV>{d{0%;_OK?Zo8vkDo$5)!(sxR)-b)7*>S9+_I!p)(>oy)K zqA26^ug1K7{hHNTRoRnMTUPhnr_Y~*PD?XsQdpU~8kQy27?#ZiomP@LAbEU?W%;a3 z?3Xs=C4SXMpOLFd_XG34)U7Rih|WlQUZtYvwdI3zlw!<>M)O3UJuK1$Jb?^;{1rq)e3A=eb$fcYEuPhBQ4`^g)tAh`=RAByxFRC z;?|iI9%aeW=Ks{Udm8X!8HXtsj$(DdUIEhldtouMzc$>Ib6h7y0))cEJb#V z^*Aq7=zoWYzE#OF1-9L-T#sBw3eB$;m@fWW-o+4(bh%UX8s-o`U++*S`=jH<&z~%K z$#YUs$T}!hh@IJ(6G^z#cBRuf_6u)wYC{($@0);X@u-~zO)xBkbSXQZ%Bm&L-3e3W zg@)HX+7`5@rHd7xUpbw!FQ2^K`TMt&#f-X}7-Mhya#wEk`6TwY zDh>|(8}FsI?*e`LG8U@C#4i)RlA2)ye;4_kEmHrV{Np*P6x_W%)`M&*jGK?yA)Veywm_@XrJ}#;n>oezNirw|e zq!vo+V_j0~v#ws-xT{)?-;i}Gu(kFlt*MxQcXTFEveVh(>JR!%rr7oLPz4@ll#q8O zj!h^aW1#&v=2Ua_iHBlvF*CVaedYV2cy_AAo6DxDBbPrE*j|&uojqFDB>&kYc44zc z`nPzr{Y1!!O-|Ic(WN10enuE#@7Lzv>(^FCyCZ+cxZzY}EQSRpX*pxpe?QNpwxP6r z_D~0hTD7lW2xc^AU5IkKD#S-5Z+XLu-K2U66N#B$V#t2DlXgq zj8r*JDYzAE&N^G|Fn)(h?CVuUPid9mc)8pdFU_J+JJjDEQ_jMb$@Q(q_jairuj?&F znCZ)Om0?HHPi#MKQ`Q!JM`_@RqT8v~y%3GZ3_09u+$qi6(q*$VbC|4auPR*#wxr?u zjTN}v;M_e~<1no&iz&BZpR&98(YpFpfo&MhE63KbnBgVtNv`4UF5;@eD_<1qOHvEi zGCr)NK9VuI*QmG;+5%e%?p+-=_{mp*xIwdT1xeC=dB$SvOA)VWOw6k&Br6g9GWo!A1S>)%tbK(p zE!&l)^v&cI4WZf$bmn_w@8x`}H}CV@qS)zzZ&o~S=k|`G*;Vc!jEFq=Tb0zTvNPl3 zw8NR9wGuo{5Wi|rzuPn~tT-tuC6!gvxx9IVHp}XfIS9`wdU6YYEF{6#z@;N^#c_PJ zrCh4`A^hQx2~x*e@)E2wfxd;af!F3mGI62h_-^q(4Gkh!W6HgVtYdm@YRk1%o(qd|IhN)HT#wCr9TM>_fDPI=DyB{USb{Tk_t zQ~Dq~#ugDZ&&r|&Q$H`m%*gN@EhReVyU* z{JoXWXMYd@8w}52*0wK>(Hl0DZ;J<`V>gDGEGK$ z*SzYQL0l1{b(zXeuA}xPCbjv=`vIc`6kfi%M@>4r!Yj>>b|oA%jngmpsQj^!*1cu1 zUn!km59_OBFNe-5aI`>|X-&MJE>e!I=SnVTtX%HRsQHk2VN`BPfi3$ZF(@Cy0@v5R zhT<{ZO8;3IHhb6lZkL!PIQij?@;hGq&hPzlapd3fx-uYIE-J|HeFzdhz;)SEl@%vo3^Asbfu#882K_ zn||!m0DfB-;W8GL;Tp?#4+XaitJ~ynhZLQb{^qKhVvxUXGa~*+I7;yYiQJG$Hg`pn z%R*M@yl}&J3isd1bu)O|aL>d%I)wz|k~764INb{Dn~8E;tr4Y>mJOS7ZTi#FSgyQW zhZkoM;ex2O@tXGV6}-+(G~)*R6(!3hb1Y$JC|o7f=`q-=PEPLa_@&MkYjjYzq*Q%VsK;;8y$r z=Vd}tQDo^$4sW3L(qvhRN5Na}18&wZ$mM-}?`snZaAQ@eCFOwPZLXZUnQ!hHv953s zR>0cYnOq|EC^5xUG5NT231KC;QU7mVtBc)KPv8wdWri+4wo5KZi4TLvm?Oh2*4T5} z-4BO4XyJ2pb%m{a;OGO#ZTnZp7<_r$?9sYsL+d^G9Qt)0-A8Rjj+)&m*XY&8jg}Nb z-Ds0zty!TVs_Ccx==mf6gEfxbX-~5s{P6a?${m&8a4jlqs>2KK+eqUicE0D@&+Kp- z1yX(zQ1u!uG1ajnB4^HXib8i(?VO8qzCAf-RcREcI?-~l&aG$W=Wbg;_E>cA_+-~` zjmeY5Fe1zw&7%rBS)fYaY4rwy2$wq`K`6sy-5Dgg=cP%Zc;$4}`}a(}iMM;LwjB^3 z-ExmI#3#%aA_-J4HB6)K*%{?GG&B^CuRo>bbpuFUrW>bQ!Umtq9eit8CTGBixG(cG zfbifToLS>tkTlJ*+-Kya&>T*KGooS4l|0HpO#fNjIlC=+FRiccO!J6^nK?}&H9aER zt)_J^`{+zT(h6&Mj=~ZS2f(`c=TH3W7{`zhg)65O=gUc-!p25uS`guh!eesq-q?#1 z+ujnlyiRGfH1A9(w``X=v6T4p&CrS^3I5*)6@;@@jZoxke?*$YGLdfB=BLIN4*E z1j=#0@cgB{MZN%oamG}C*EBbiiq$D7K#OBWP6+TI;<@+4nXS`>2R< zrIL*qu@h`qt?FHP`;hc1*1gej34OosX0KMQzHFwCncpn+0AXRxDWDPVxg4*{YpOcz zZOQ})#i}=vlJ1HIKl~8AYZw_MNE=!8@DmM8qx1nusTf(+63rvom>U3=v7@(%pAj{aWrwMlXc;Z|LYO>qn zu8xn>=u~C@uF(0Qg`CS@?+D(}$frz0O;U9_b0x_gnmJ@x zraNF4>d#^~A{vwaPgB{=X~N*}uBipJf*Lvv-P+Ca z8I?t6ZjNLoT4iP*&FIDK?CNfRN!p$%pbpds|4`H*rAfiYUFe;eWPBg(!h`VgS*`3` z2;(kSXi>{*l)j=+rpw*zt3u>>af9o(9v-BTO_)*0)}F8*D~+))Zz#BfdQd1$$)%%A zI=u0OCU`Stz?WU5qGV@g1=AwHt!<$aF)v&h@&!PWMlH^-2b_y{8^lyb=tRZQK722;vwge4Zq86$cA$5_}EXnuw*(dD#w|FGvy!_?)@P9fS& z&Ady`NfmduXx;i%Hq&WkMVF)Ih2cIM%sH0J87F2BE#4hn=D5bRPrF0lE>Zm{kKHbh zk_1TKPtxhEB$6oS)r$T#gT`(gPB^E37eDj)~>b#e!L#p-A^0ZT4%jy$zK#t)8{jdFR z2_DBqxhIpf3twJkooCSY5V|nz#uNV{&u5v&+VXbv)s}vHC7Yp)Z3#~5wGGjl)|%E~ zyYB<)k+js~>^yQb!G2FTSM;?u>?@Qaw+?31uWqcdn&m2WK9FLa9?)W;xBXFVWx9OI zGnRV4ea6zdJ>P=6MBZbHM9Ev=7QQ*z?G#?b)<}AW^PEYK0wyWbpXK?A zXs&m2X9Lw%W6KWaEdN{r*~IMcZ^ujFJDx67arBqbD@8q9Z_O9n4-29POJ^Iu=Ltmb z-Wwo2PwAAI;XKc@?4MYUuMDYSC^e_)U?zVv&|lqCLY+U$e0KTeut1-1RF;uKc6A~8 zqARIubb=5wA;H$5_5t-m!6Kjb>mZgNxSGwyCyBf!O6I`Vfnvv|7IWLuhHhx zo_k(pe{M^RkNtOx+i{fh-0eB~Ov|P?xuGK-_aEw6h-Eo5)wzLQBsIloiN+9Bb#6`g!}a&V_MczU#F5KIPbs! z{YfQ2cv-HMvyGk=J*a1usFXl^B)sk?eO((}AV)sZ^}ygax)Uv`o)<y}%wC1$Igd!uvB=Y($ApxzL5d|BND zdNbUj=(*6OL-il^zgz0udzl5r&r29>7!@!!y+wpxw2P{A0F=siM}1hH`c^xg<$i)T z+cejB-!PKUJpaM(Mdg-_&g(p*>+$rw9`RBG&ks!QqIC^)JD&O!Zl6Y}3{zZcaYECnU zaa`KfSxXK2@$FcRam}g4y7_3-T|$IR)HXHopi-bnRRc@WUWWqhEf!Z=U_0lPDUham zA?Utct)a`Bvxy#PMEZHLZ`$7Mz>o;WCmFft9}KGR-<`DxzuiM}I=l{mPOV{? z%>&)|HZrNE5v&<{#JzIdAuPnrZ%n7L;IY6zP$FyEqh@)o_JXa>@^`#E+boyJ)bsgB zANKb;3eScaZcL);Ggj+o{S|R93T$6@4y*6$4+=!)HC}Wxp*F#C`)L&bK`+NA`o~q# z=&W483+3?J=WwQm%`Ydl9_Alz6-ueLsF`Vq`Gken4M-kzNgk}m`5p!Q-3{0zWc_W5 z*H6sZvRON5aj5EMuUalh(_Kjo8XP>PDpbN`=w@kJ$iN9 zAD(!_wY+pa6Q@;y<$hFGHu1`|)1x)^4|?1t%g=H?QpnM#RB2dx-`z2rPI0$dKs65F z2Adfk!wa;k9G_qxU$xX-q!qt4i{MFAGL_j+vP<~nZjzFg7RMRJ8uer^`tP4;%bH@> zFSg%BK6-`ZDDd=>w;@4E@tOaa};wCKe5Rm!z*OiY9_zM*5+-9qiI zW}KJ4(ls@e4q~x0{jLN3aLH5M=7<}d4}fCEX%R_f?Uo;cX#= z*UjbrZw+U<072@SBJaz_YoeQzN8LwzvOt0B_TKmH&%JiDxd@(edb?NU5m%pQuMkDn z#{e47eXvcnnDk(aTkk)3Q$;oWd6jut?p;Ct*7@-;B?^wp%faP%5P|@LXC8P)O@<;v zB^*n1JIEf}<{ZhI>XsYdB2tUZ?HQkq+j^#6R-sq7J=Pm=YJF3#Jhy>?$t7Z^78IRc z@)yNDPFH)SxZ!N%aa{_C%BD0Br$M=tG&*a0HptNCaS6<$fGo zw?5sXcSkLj6oJr?fP;0!3L2)He*C)Y2Wvu^ysLlEQc#OG_U}EKmvV8V1Ut5^8pY_( za2)LUrZ5%;PYT&PLPAN(Ja9T1A_)!ssX`$;7J0Fxy&cpts_P(Gqw*SlN^l%RvfHPQ zC92>gF3zgs&Nq9j=uj1XnM6h?mD7;Gr(+!{LS8+-msh{^%tVjA(^gq)bY;%&=Cxq~ zWs!TqDFYJAUX$VZa{p0M%I6L$Sgc2YfdE=E*aXQpNWThCm?1c+6+O>7s=eqwAH?KJ z_LXZI;SmMrWG~F}_ts)Jv#y$sP7PQWR>M+Rp&vffp_0*Ljg-rcK)#59Wrav&1{0`3 zLukO@LLJEavYPph??<0*Dsp77b~1RT!zO4>qlAd9of5a*$%_eDI&uT>BAT}0~}KR+Ki?^%ev z^k!cYQN#>`tbCp2|8dcLdW!bmE5A;~8=Z+4J2i~&H9M|q`g^ni1{06@zp{De{YZ=( z*^ftA``Y6-SzRq&GW|5X;5~7jNQn{|g1^`G&6$3r_*Zod^_cUN0x&e7Au2+ZmVlfX7_RA~Z(OC3fNCUWX5ZXXy^8epC4 zN6-PAGDEiqwqL;j7TESU1sqJ)oh|yACgT3?iUBrm#=s@P|9DXTMuN|J1IIXH5m&RB z(!PYly)+lBiDxKk(!xL&8yRM}h`|!Pl%wrnoq-y(d(j7l2)9YKp4}_9i+;XMHY$CP zb^}=0%`oPM`6#i44mu|l#!0)9GRu;(IxaTGe_A{5cxDZOzGX4dXklm4iI1&LQ{vBm zFPWK5aA=$tP~}@VJ3dW{?hMZvgHU}g?o8*X`7g}mVo`B>O*pU00^7=RyfaO3LkDbp z2Q6dSxHe=21CgHL_jW1R{^zfNpF(K*iaTgGT$s%8Xd$ z=sAG!M9qI#Gkj-jI0asf3HHfVIEMX!{IK5T|Dg%Z5u5rpQn4?5KI6R?Mf7~2qKvsq zG#u}G`wy!MMS6ZWf1{LM(wsN&i75AwK)-5SwN9aaPE2q+!v#IXM58vnY zYbmYLOO7lv+S(2yk3=Un?-bo7U$b8Cf0BOI88E0gUDMV;YJovlH&X4C=-0?-jIi3A z(=9^B`BAxlC-9_~5}J^37y@e%oG6lJ#T0NO0guXF9qKlzAmnr{_-JqpXXA8A(Pf-> zo$1+MFpOwwXp?^xX%fC0=cc~Nj0}B=nj&d`ANJvo>dFG0Tkg5) zpDLKNK5HK4%8*MLZY8r1=Q3dKCuU1$Uo78Q@rtprxVP^1wp71Y^q#z5)>(Iog}wS` zq|r9)vFlAsv_&o*n%-5TuODmhjpp%bwQZ^C{_sT4rgh%Ue*@dZ|FQt*PrIriFU77e z52m&!hoaw0{eud}hA*=4_W(rBmNDP>k`k={_;pPSeqSoL3!1LA5@)o;74 z4`eVa8npa=euJ9U>8Alc0XNV#u`Hv*iH{4!{@WCGO}l!+A+Km8Q4`X-a@_ z@;i)@8y4f4^HT6o1-Lk6ZmIeA7CX_QTtY(J=X>yaE zWh2l`kFZZO4|#mcqdU5Tqm3B)7E%4Hn>E7aH+SO+20`7WbOepebYOCUNL=WNHNLeC z z$Pa1RH_ffQpS@noq?q6xcwlk~Jr+nB_J-t4$Ga%LVF8Y@>dV@cYfjbiG(PPQ55n8E zHkw6e4jAs5chsa+6`Z(4>sO{l*=|9OtLpz@9=pL~B1d_+UkhD6CD^eML#1Yo0Qh9?eEG*lPlJTu%3IrFH|%L zQ+hozY~JqPewf{AujF2EOiPh!+h>kUTszU-nY&#-6}xc$i6*ycVR72k$G38h%BUvH zB9t>)GdiP`x+oKbb(d-IN!ofhU#Q}#?M)k*B69U)rANhBl2VmGhPdSvK9AEqsUsmk zk3>*C6jU%kk32gW2Ppy`h9A|057n`b3GYG6IyCy4zQLR)ZvuwsmXe-kUfc$m|F4$` zrq4^ObF=Lju2(-kRs4bJJkioGumz|-nK}GV;{-0K<%UQM0uh&}SQXkD&f;F=hk<*} z2owtpp1(es93YwuoRj{xQ)`?0x{O0=LzG z#4d0@0}uU_p-M*g5zWn?xzS2A!KlvDn>u@WMl$eOD6c_GKfU|wQ2dv2|70V0uBR4Q znVJO}$RChsXZ_!ZgP9pnI3TMQ-3-eWJo?dvIe~Vcjpp4%h7Jv&5%v|$b*o70A#ba5 zNCj^Fd*yXcDhzYao{Txp6PG_Qq%nt&sbNa%R zfa)XPS?bwcwlHR>+rG1W9FHH%^|d?GCtbf!A)AtadAai?(-o|HTx?`%mL@e-4=W;q zD~uUn-kh+S(Ew)SU=CCM%3R;V?LHLge-Q>Aqq@?P)06HdV7LuBxs#UMaYVFpN-Q(x zRew4cC=TfW7+Bj>#m)Z?TApWeV9Ftu0yGUCEI~G>kHQc*K_o}?BqbRovH^d#GKl|- z=!&#Wg?I<(svOa;&6DH5+odM5-l#RaeO(rj$OU)=!JfMbC`dz4lj480_zpS1XM`nq z9f7i1t}UbJ_g`(#fca4f2R?OQ2N)@YGJxi{Avs0P!Oy7yYSO%%P>S@MeGaW0_AQBRre&CY$+ue#L+%Bf6@1#;wy(u>oT1A970#Yn2>98mM}B3W>n+ zdEZ=?OPh3RHSw5omiPKWV#X?|~`1yuFZ9uae91^k$lDG@lX1(Q&>s-{v z&t6RV?M~<5F}{o3W>*IYGBTE}M>}yK-pkGW3Koi|zYS)h)nZS=X6xUB*3B-K=y-;U zfmfZ6gYxJ?g`*bbrS>)Nm>ooktv^+MU;o{L1c)a$w>Sb}JqAO`&deMwx&O2~20=rK z<0INe_?}*~T@1Dxm^izWSFh0m-``EXMWNQcNO)p<&EGKSdOSxUzSh75VVO6l`osd4 zKkbn$xjmE_OGuL19(lBRv=LyYCbCG;u)k6Q@se(w)ZaarB*fz-6EGI5j&>${l}$He zs~(;}GWVOkbz^aceXxfJbD5dpzO$N^1S@WdFeB!lAO7xKeD{XY&S#al0vz6-Nb=9_ z@!xHn?FLPU)tSB*9Q4J>+2cB56v3rm1Q*Xh8+bxEAm+6gAM1MauIV`3g^Z=ikl)XD zZxZRR20EvE&Um?iuUsqPu|2BCkztQOFUKnL#whbP$jHdRRV8&1FN8SkO^dgkZi(DM zyd*9GWBN|#acSc7fp||*`Ih$ItN-_>7Kyj<|M3Hl39rkc6j5Ucj>@iuT&>Oi0B)(mJlAaW0D6hB9NlQ_^1Y#3077HDt0UdQ~z~Gr1;{6s>4@BT+5Wo_xtk_F2 z<>9uej|9!U4R!Z**?yCdkUabwVc6ZIXDf<$4C)$0MUhbGa9%@pwRhqgK$Ey<2JSLo ziJGg6oMGPVnRO`7GGo&vT^l>fKc!ebmhWvHV2{Gn7>=48dMZ$0A#($P4kL~N1-nU0 zy5I^LY_=(2YwgI0&%2*>eM0P06Gd(qT29Z#sdeGEF(1l}p3A=o6Aa7AWSh&xU1o^e zPZLyS`EfRo{~ualIE_H)(*i{|4&gq6z#yOkfwaC%so_0+3RyVgmt8-|jAGlzpo;cZ zL4x$Rcm;P+>vD& zou1V`0xNHI7&6EguLb{*@sGAmg?&|k_A>vaCxf}o<#@;d0TQxyc$s-4c!g6Tscf=N zV$=Woau!(|a`&i)W82`w$I>VN)?5CTLRARZt90I#wM^$N|E1%)=x^`%wQdl?%~K0N z^&b>K<-Ptf(u5KuGd&`O^uc2_NIUD3pY7{rT=EItV~7iL|K_{FOl^4zvG67a7WsSD zJe-eFVgl;`b6el!`mQfy4y?bKt>^-hyu&M-^Zfs=@extjRzFIrDdSidmjlP)7Mr;)>+41xr2fo;Co z4dk?bGoT?JgVcg<0;D$g2vUCu3za!bC^DAj-TDICe2W-$;sJhD%8qnUX|&9Gb^C-Xzq$abAd#@0s?THL<(&k*69ZmeM)h(EqX=56Mlf z3vK?}`aH3Y*$=%PZ$KLQw|I2J5JG05Vbl4ltm&h+3 z!FqMe=?9|mWB~klOqNBa!3^IE0JSK_QS>MBLWJ9srDBqDD+7zkdlz70K3;myqfFna z7;6s((cmfo0oaAg^3uz`yw_))PByTc@qk3d(oRYN&;jDUX+R+hVZlj-I@F|MEGiY& zWX2RXx}Ew(JJWK9iWc>^f0Dn_8$^VPiM;Qulg|_}lWk^*?0LgBA&y()&;I- ztxaZ(Z3ZwbgjxLxPpJq&5}HCsY5kxD9=r1J+lYt@ zlWgY%y&8oq`RNys{d3(?NC>neMh~={N?)2$ON=(aF7_m0|d_+YQyNtX%MS9q_4bnV4()C0@U3Sa7luC z-Vn(-wKiW1P_{r6$_TT6SB^(<&lKe2JW2vqqY#ME2SbrfF>cgMVa!To94c$YWmNSV ze{~pWnPVdyN{*;rC~95+bGK?(O{|-@ag{mPT>GB1c6VJ!P1zU2_NbGi*OxKn#NZm` z)#Vk966VReyYh{r%;L)dh(J5Qg`jSdAd9xHm@1p$c0JC-~{^%#2o2s}lqFf+L|s|Ar!*wR_t-=q86 z(7tC8NQi{SaKx^kyo9-KFF=dRe-jZ#_P-t4Q=;6~@Xi2TGVAQ_ftsF%kDu;MY3 zbXfq~kLgIRQDVUUK(7L;9P$ivNvT}1g}>bV+IKgtOkKC2?*==PJ)d9LeHkS*Jy`N7 zymkmaX(rBHqJn&?R6m>tx8X_uy>6Pa5j6>n4>C4&C>wuLbd%K4K??pd0`k-UrY28` zjGBrn`1rb@R-!vWq!;X~rokP*tcEF?NzMzU<#>r}ADPL&loB0^FdsrNAH_p$=Dk9>oJw+8GT$4;`sz>@g0k(y zk%Dh1^&_MSbw1bQ5lb;lq^b4I`d(jm)tvMa1)Y4p5nj$^xQYB_6}xk`IVwbdQ&G{y8=bD7k;H(uEGtzEq}3tyk;p(* z@K!XD-TjhRCN%_N(h5?%W=OsLHA+FQg&a-Pd;|j?Spnj3sjcCv*G!kxhOinBv&0jf z0Z&r`)sv}R>!HpQ==s-tzUECHYZFy%2o22}hyR_M@8xYQT}8fs72~FU!UWrJ;eS*n#x-mAdVPxkM$|Hn5LS=%-QaSw z%of*i5CT)Pp_VhyH^z;{i7uYuBhOyjjbr4{ZV8<+BCc}UW3v12LBL;P^TLl&7OoRp z#MBu%MD=%k)_sqG>@_>Q|8&*`IDqcDXm+_OxIbVEM+(j8XFi|1;7rfq+75sfbP{KJ z`8i=;aZ_R^j5}yTP*cQ@|IBZ41Z;B5c`@q9+*N`o)uo5NvlU;Rv+)>6Y=On)f)ht7 z)=#;F|6>ed5%*Z9yXXKlT7S+@GUZ~9RTd#|o?V+Q%QS2BD1*t9D%$F+pP2Q5N=pqm z*+M6`8&Z3csimS9qrgt_T1v4|k|Nb8waN67TW|e)ykI}tE%+28GdMDqqZ49916WqI zlyep+nJQchsn{xVuS;H%nM`6!WBg`lNY`SL8L!aHbC!SG_T|Uy4h@Tms7K~O zoTR6k&@a|&>a#ZLY4TdpAzFwR$`(3;OF3))Bh%l#tiQM3(7S{X({nDs=zD?Zv86e+ zG_2N5I=bwP+ayt8VHW&EetkT(Xm-44wvg`H9|NqmzDEBaTuxR5A1OHA1mIw3e&1tt`RRG8nXukL@FB*VB1lf|!r=DXIm#xb0zOeK8^W)88 zUYuGico8Zy`(yR&EYoxj?_Z=p`+UI#h+Z#)=hRJbZ3=yHVzZD>%Q{-M+5ABkpdTf& zL^BSN@Wstuz9dR+U5cIkWK{s=8h2EOxnk1Q!cr?pwH3Y1B@%=St{DlMJdiLe8Ff+h z&E@P-Ss&fbw#$$$-HO&s2gj_+ch6<~X>O!tU#g%>*UdMq1JmwU6K3Yk6}?tN;HLB< zOUX|bs2?A&XB30j;zUYlJY4g(}ziq?E2 zx89LKvU?L4vaXB_(dibNdiK%*UrnAoJL!zPRI;)%LHMoV?-R2Fy z&4dbHS;@_q=q^Z1+^+EIv%k~2OB`F`VkU+N025+kJBgY@1H?ZTY@`lS{;iV51idH* zY+3yb*E#xUf&chC=qFv*ZPGtEM`#f{2?JB&)yjuiFl~}jrQpg|@xbDTe5;{^8OlD% z8Z5Eg9s@$1!Sapf!zH2CtC0E0en}d;*qg+`GJVvvz{VG&?mKr40y)Gr5TjF~+YOqJ zjLZUM#q-zc;C5~jw0##OFrOeL8m|em+{tRPxU!vL3OS-%*x%betsML#Jr8 z%Z4JfA9}ac19k+<4|c_Y-CU(^LDdppuX}_#{xHITlo@eE;aSQoHK>-G*=^a4+#&>Q z>jkh;o4Ca^!B8b!Qxld3RYrJkZy`t)|n+_Hy>?wRQEc$6tp*4Vnae#nq* z;5sOY*O5E;b0TTP-K3HyobgjoQ7nh32;#BmrMp>;VXp7|X4Q8X^|a+5K(P9>^rqmq zfur47J|giVt_o7A!(R_f2`DF@w11*uhfs^=3`n58Hj;$vUMt0gZ~+nv z@khjnkoC+xuIN0U;G*DKFhII6r@uL;=e;Vw7W!i$Px+8G61xA`Y7HZIX%Gc6IN$Ya z>f)L$M{7=c9%Ojkl7#Vh<+iCmF=_`dV^$&1Uj>3{XcuPf6Vds9V7g0ePqErw8LjeI zGbGOcA;geMLPEqhyau3KY|-g>$;lEXjE|M-F+J<_-2w=hZ5#lTpa~wVJy+!I>*&a< zNN$_lS%|&>Q9Cdu2;!g>-1>bAp>2j)IE3Y$*)-3vQMAf9hIVU8f+)mX(5F~pZKm4y zCc4NA@sHGEh|_Ry;%V*!GrDAo*!AnVwuVMk*l}(rgG9)LfRqczCK5Sko~V_DpFF=y zuy|(X@CgDXLW2?73&5mFOyJf_lyvgphqxDHw15U7V~K7#RHH0_QB<#C0Nr_0b7H07 zID4kcot45s4_#l-)}B`{8Re4XM0|{eR*}2)0*~-vDo<=qAXK;kQbMT3?9XRvAghea z6X@Isefmz&WMi*>-*RQfFZq@PFl#l$Hq>y2EdWA!Z|pH{mET%_wB=1EsG9401A%k~ z?oeQhLggBoPYd)Z)vqDmkGzF;z_O?NIn>Px{`% zA^vw~L;t8Gl`|~1S(dne?hQu!$2a6FQx^5S@EAXDAb!JQCu8(KBY7gY@{KtP^AHFG ziCqPdU|^RIC%c$@)5&`{E%q9Js3%Quw^URBf|F5k_f>J(JD%Q^h8)}L z7jl#`#eC{@QX0JdYWytf}C{ zMXbtfTIMt!)8E~BpK4hK70JC8d!iLx@&a9;TltR<(RO%>8BCJfgz za=VtpPR3aD?hSWJds}_qqB4gTmu>|=9OJT;2L&QmBjZ^HR%Fdx5CHP{Fb=LMBt}+%V2er$kV@R^7%|Cz2Xu2?o~bCmNyY8wU^j;5K$<`;j=wC5wQwh3MU>(_gM<~mSVIg*zC~^x zA}iJG;{DU-A16OA!62u!D(yzOh?>1}UxntWjbs_I)NL{EqnUmh_QAyH%^9_mkc}LfYUw8vI7+6CL66!g zHn?J5J=f1*Y0tRuUpA3pS3(l$v8zdad|J06YsgEgY&z=~81pAO8>kAe%(mn&&d3Nf z&n2w#mD35~x8US1-E2u+_0SfhRUNLJ20Ypk zWnW1Z#}swPsS$h9J^UTl#gUb@m1+jo=Aiw&sF&szWh=f;?&fGMl??53-Rx-&% z7YZ?&6^1DboEPSb2z3O7XTVx}jtQdym-d{ix-4 zq|;UzvRf=sQ!I&ykQ?@v(tJ#*(K4uTY{|O{!_Gk@+iwIy`!$yjqxNwr`GR7-6XY~m zmNOdOn$y6zE-PPm-+^eHlN4JhQ#uEx}Ph)-Le11Xgd8UnVtN)QvxN~Y4~ zr}^9VaXM<671+2=M!zjhgIc@MX93@;?@c{>xVKZ^n&FE|wO*1)He@pJ8~8v{8SCcR zwSPZ+gzv;^o+Ev==mj)$1`{I?3Svbbcc|{W6PLb4zb_t_?O$1NEr?ht86n4<*{zWx z8Vwe>lGlJ}LJAX_wZ!aXS^<}WHU~69UsFX4Gh8_mP12@RZ4I`L$3HO8PSM@(xQ}%Q zunC<>MxAfHH*`7&^Tce27_LKjVkfgikdsJ4be13+;3LJkSB)})5W5j?wMF|yKnCqO zPc%?ITdOCnS4=_vW6Ss+28zRUH2UC%)i+V;m0gc0k1U{E5C1Y`{7WieM7c(~d~lCH z>sx$*UnByZL?D8|ivrsYc=5m14Rd`%V|axruW$d)%M?Tru-%BsGpNxUj{+W29szp} zAr@e3NgPorNeBx_L)C>!?KTV~K#&6T({15C@?i5-VKw;hdKUger@Bq)d{fej@rb}} zP(EPBSL~l>*8kM+OxTumm%R5=IC;T2C+gI&z}@*S?=^IWP%>kZiHAWG06=0kgNll3 zj@ZhuibUq~HY~*c`7Hm*a;0t~) z!vPgY$p;tV*9r^^bYid?Lwg$4;|{&bRdaN~J!Yae9<@#i^yS^&73YJb-JT@eC9X4} zKVxV^N3=DSV#&>>%U<$}MIQCdo*eH~c8SiLWJXTSwGJPC1f!xG%8mYfwrnP*_gx^E z&Ti(^ygjx#l+;pF^IQ%Bdy8_bRZBkV-oMeBy9zhhbz|E5c6nViKT6~8Q?TKa@xDAq zGuCS~$k4qRw2)!nEL_B0@Nh~zATv?b?#u&Lt=_n~b}*X!5!a=-fCV5FcPF<22Bh%t9@sZ#523WCcvtlp!S&B4v{ZexeH5xtJ z^GSzVR@0Uw&Rujq;;!Y|M6K1E%7uj(rUJda%mv2fm*ihedkiTw)@ZWv-fX&OH{=C# znyIFHSMsipd@R0oD{clT&$mgSWyXK(P5t*4fp2dXHwb0LgHmq*iUIb?vf4ZVT$K&w zSrV^Ijsn{#_W%l6%^^u{yFQi}+=Qc`iNkl3cu`!i4Oo)?cLY(yEc&*)iIa~5Kk*qj zeuu}v7P8>$ugSVEwGJ00x>EE9%o}dDa{qGZoBM3qE6`Wj+gVZss-u6!9oq;usS?wQ`t9VR zOgh+o*bGQ0iu60zwoh!v>nN2Lp}vjm=GWUk+MId7%WYblpKfK4Vq{*_%HfyMHDSTK zfmgsOID7vl(=lhNDu2^7YInI=$Zw|7PGV;Rma1!bhui3VWq&Bag zT9CoX+}wO_m|R)MV=Q|7lY-Dbq?KkS#Hp(EOU>oR!x=+w1k<86^5#l|%x8q$c~mSk zX&#f)EN7N(os5BBZ1C6YH9laFw|Q&mIXwEIphbQE^Wq#MCtGlZ-e(IleQMVv|Yifebh%WJPCz zvh|0kfcX*|8lfLb^$_lfzdm`t0kR--tp|;*;j?@Fo*hEdZjWy9XYbQMH-t!PE%+Te zIi38y&|70AG8e)ENl2RTje`_|&&XjO*ViAkHUH_E66!khm%QDz2#M%#Vlz&Xo@0l8 zOQ=)I>fR)>IyWs-t+A3V)sO!k?IsD@k9B1ojc0We1t(%`-@KyPd)v;7sEGmX`^!s= zJ@~FX%8Ko~H>n~cLzEVYrQZq&UlMVGK*dTAZP@`?ivt=>-%LRQ#${ATuNyKZgO(>| zocsxCV5lDc>6A*`fS860yrCOn>3ZXaBuF>JRU6td+6&D}JRF(N zbLgGFT&;3hRBD(Dv@mzuxVg09_A*-XV`(K4tRoQ9j zw<2qKVcedD)YD*f*3Ed^k3-*Rtn5(F%p1o0lG-9(xOno~5Um92(67Dw_zFH^qrnsi zi7qf0|I#(He>#L*e42sJn-R6L?of_Q&T}_>-y8p5-Mx7@lz;d(JeEjFdl6cZERj81 zB$cvN$i8IXm(0jcicqp-&yp-g5Z%0Qt zFrT^SzCX)#o!5Du7m&@Df+cI-kv9(5|IZnR| zqg3n1Zau8ui2VWgIF{Uk%I*>El6`>LD<>!*V1cR0?flHAHSxxBsMHcEZ?%J7iVtD! zJWize0><11^|^^I*l%{zeY=6&z-w3pN=JY@hkLr2(2o*hisO6x}UlTa{Z+`YORL+TFL9yB3Lz1zviY9 z^MLv-Vtp*a(JHK2x^s0lGsfcmD~Y5*N^>vbl;FM@GYhhRwH0pu)KM4e_5C3>U6;`^ly@|{s*+fU@eLCNA-->*QT14Qz zlIGL=$RmfHlg@mr6PcZe4b`=+Sq+4i6atSt%0J}P>e>a zTIvY@Ee_XbZ~ZoAcTuCN=~RA@){%7Uk7B_#L1A`o6$kbe;z@_;0|8P1LD$=cIMnJ0 zX2hntkXW8`F<^`n*rskkg#)C^$X-?}P@iT{n0W}uag#`8_r>igCmL-D#}sgqHwf1022VMHbb`a@_vc-X8(>G5s~2&32+*E5N(D(|tIX)S zdetJ1JW;dU8KdF9(N_xVJ^V!r6i0Z0$QF;_R1o$Y0V4p3A&VI0=CVIc?Cs6+YgdqU zif9w!VYgH&1kZxh(KK|v95PD~AQhPur`HC6>PT9hd2f~k5tJ;gO7 zo7T1LDc|)2qdI5^m?>XQ)-X=+;UW6j#th0PSFnsJ5i|U<#r5sVCk91@A&FuoFUr0~ zVbWSLOIEk?E6(s!C-9assVa0u=WpS?MSn}%;k}%t$D_)e6yJ9 zrLr73!IW9!I7Yb{#U<~0s<#`=y%rZK0mG2(;t)AiKdOiS!(AW~SzzbSBP($dcBS}h zTYcAJPuBnawghM?0eNPZ1q;Fd*4i(_gAeA+fO3!*^Ap=f-@zh0{m)q zyQeLp{;7f>WU?WUPV9111Y#+&$EZ1P^LF^&4s0BF$%%c&N4wzy=-S__p&W_ojFq_w z?>cz;Vou3ev7eYbYk0h*$@T5x3DEzqc*(hPSUXJ9_KEd*$6D%@v-W=opZ9WJNLuhr zg^6m$nmrwm`nuYK5{GOxEF4mra|$=BNpX_3vubWo+;0W&hNcSYiIaV4+wL26g={-O zOT4y=_3L$c#UU$Vsa3=8J`zj!{_0I!M6hx=<)5u^K-n(1(QKn?Mk?aPi+`Pzf9lfg zCexM{HZO9i0(lqu#BQwx%5BbLW4ip-umgLXOY?E}UWAbt)7oz_` z1UcHA48@V2YbiqG)^l}uW$ODe7pXt1e2)Y-5;t(RNue)}HBD^#Jz~8HM5LRGqjK#o zQmQvBCfFoYB7D?=V;H?fb7NA(i2PsY}s z{1(lnGNxLqbAv6o=TrG?-Xqo~-Qy1`i3RSCHJ$Eh?!79H^y1#SHnF*FE>z}V$6kB& zzvm0J*B_v-r7w2atpMEB!LU{G6uWZVjpj0EvXl6%7Hu(iKP4mZsb!q2HRG9;1P4Mc zS8`jop zeTHrG`GsgaJ)XK%Laj{xX~np&tYlK1VV{@|@XpbsthH!f@AULg_gi8ADEXrl0?!}$ zv1dmouDteC9WZ;*67~4QDCKocHr>e;+0UiGihIyMsk9H(ZP=cWGLWq?Y$%Y{U^QCl z=zWW6QN8_>b!@FrUnyK{XT?I)O+Up${Z5r(Tq?ONiZ>#}*j|n6?%Iz5x80Q^WI^oS zK7>yeaC~raKA)#J*z8y#I(2nh-TZi>f4Zs8khh*L#rN51=l0S}l2Z<;iY)S&Xjg6* z<4%tA-}1Xgx~$6R8ej?5g+R!9=V@&nbT)F=`9)2&BB!=KUw4{~H$i)?ZeKoC{(5CY zZ?-#H<&*MUta~`erB=eiKs49KiJEU#XOY)`XviGWQ!|xA2!ee(Hz$w&__t}9Af_pt zH(!gqJ|lv=7kw8OsJ-iP`Qf%7W(^Ue=g3Z8I!bV~zqpcn)5PL2O=<4QgPccs6aCd6;BurDMyqMr^`+$y#%4~ORjK%yzQJ-P!-g-yVJjR)fH z_R=snQEy1`RcTU{nI9KvP>YN7_-|hQ2EXS|x-+`A%tHr>=#B0fOm)`0368VKn?#;) zeeG?S1r9wDM1ZV+M^UhoJ=zzEyTurUo4m1wSnXYfYvvEPK8zM9AGv{yTb;*}9)Scp zZDWlgeUvVU4BH;{EbNCLPK%N^M|bLw%7Oi!D8a?#(SO-LAY8goG*ls&ZJV8P!I4-v zANY$&`D_+j!jwrvV!(snniOnA@lj6ySYlbWRV-w=cr3efbUi9=0X>uyW)n>(u}JKP~GoOhC+JxP7&nTXM;6IjfH zT;;1%hPDTLM;asC-1vuU*dD_a^$^Bi(MXJ_4W2`~8Q`pg-mAbZAw8Rl-%}PwuNgBd z6~V0%T_QqO*+frShhd2kTHG+#D@7B_ug%tM#$Oi2=T?8tsB$1$6_~1wAUPAL9%phJ z0TDhdN*cVup6uR5+~aW1PtnQCq zUSP4Xuzc-!h(4=;if3>izLav`%^`%v0r+Sr?AzC℞218 zmTJd;u9ooRatV>>+;-L^?P3mM8{IpFRImDG05qIvT(CiHTIX(>2+FE|LCQh zx92Hdglc4?t}0i@$#va5`Xi&sLPnfl@p~VAxsB0US(ABt7+UYuSScqyUOe*3C7q~0 zO)f*!@*n=gqn7vSq)l^HY9wLaJ-W{L-jJY{r_LLgipYNG702WTO7o+Na@lXuqNa}0 z2R=z{o<1j`KB?ky4=a>%;Bp3&*7iC6&7ZN(s&Q$7`q|3TAs*GFtODYQN{x&yvt(Z5 zMyYuUp7nB?&vXYdfMxJ9?7LtPkj-VuN&^?CeOPEqN<7{O0wQeeWoALRFQr)+wiOOxw<5DNU>`Itbvu-a+ zJYCy(-MUmgr_uKmySc{z&5IYz3_($p%8ClBZ8oGvBLG2Ixlcm5qXAT~{p!>-$$vHc`=xOPHaoHyhWZdGmGHQ*&us>h1i*k3)Vb-*c z-q`YUv=&`v`UR=~B_QCK7mTm^nrv*<0o)7;)g#BFgYzY|h^3X9vsaAIjMACh4h05$ zG8t!1c)oVGo3QG=P)J(9T)j4}+3$T$M)RJ#Mz1B8UGE7epfS!^o2_jZ445|Xd{FK! zAo<413wJC}yH`|qiYeyXm!=G4>MBQk-2`W@syX3eGafmvW8A-FH3h7C9a_q$*dp@C z4A&2o=0VtivBT@zH`{+~F?_)4bCLQWuG{i6VQyr-TUd!5IfyE*`>*oo%8|Ptm3|&V z%c38^5JF@Vt;#2j-pCgnWT2aMmxkSdJ7RoTz=XaH|Kg_I@TKYxuKJb4th<6>o5n@D zp0DmJiZYF$P7Kwn zHufuoA1Kp)qy&yY50jq{`W4qKI6B8bhHO3I3vPwX-$|oU zgMhC89aFS+c}q)6dN!pntVHbPGyozTh=uNVy9l6SiQRJOZ-k=rCCO1AEQh?0?)LA4 zJDRE6D>)B2k96-XPea0o!DDa&#=Z8LxG+lvT?H98D^iWAHVQAe4}1MV|G$k0PwCT~ zW-%?K1g|H>i#%T;K$zR-Jttd6ceX}jTOr5MbW1=*QiTZqEWz-x+Q1mKr<%!p66j($NCW^Q@5rZqpwbU~g zZN-k5WJ^WBf&@05&*&NA;(#zSQ==xBFvI3R&w+BGM_5G)2Rih-jMls;@T<$R;H3wv z76Nqi3GY}s8dNveZg?v$VK3*wMy>@r?3Sn$i>ZYqV?vWNN|PIpM5$u86L){~Lcr2j zf?P`z0FgH;;N~Us+Daa%Ex@f5z1upSrAB~bwpKFW6<`d2APa{VQt?CN^ZIw3Sc92E zlu8_{%XBXupucfXEbe8qs{qiKGwE!0)d~T@NglOe-e~TR2Z?C8KA7WisfbW<3zlC` zHUDWbo(TG^AkcnFrFe6zZe?>T)**xeW%)=AK3&797vs9H^63?Y(Q+O}cs&>x?w}}n zn45$qxjU}ANO5dF2H6MLbB&`;1tF3G(^nQT&cup7wt3j0o-Z;w5i1}aV`_AR#?WK zfOSNVg33Ra!i2Z7L#tJUPuv|rr{E+W?tiKPP5^)WCQ!xuAe@88h?)do4ne}SIh#~n zIdUu~BgR`&<-Bi7e*|mDpwFfgU7GqV=}bsn2&H+oy|lO24T9BOiicYMxK!#O2McQM zcbnvN#pbFN;i1ytFz~Ct(})tdw?T3>FYbO-n1kUXiSaskp-Y&=}s0l@-9UF ztt?=w8kd4dM)%`YRR8haxdrLSGvvv_pF7w8M|-KVdjM?ThXCo)9#hPvFUR@L?fc@oeRujf%T z3D3zVZ7vs@q*nphq7Oa`TXg_pHJ)A*z6u7MlN#F-8WYdLj-Al{-&}yB>K}xqwL%k} zRNWBNgnBqnfH@lk7|JV)-AcYdu~8WYJ(7+ZXJ7VQ`ECb zH1(dVJ1To@V^2~zTyi0)+N1s6EzVd!e}wB8`_T9jVywaca-X(&Hv4|Wae5LEcHF;d z|847mcPQOM1fgz2xE@gwk_`Co_zhKiI1ju_zy96b{9OIN_IiQy=dS@Nuh*Zk`~SJ_ z-@sVBwMH2Q2_dKryC!K4J_(Q=XX}c*w(IREo&Ulp->vY!KksiM zV3MHvtGR(tZxEgz%Px}=(8WlAW(()KHP5xRmqa3-C%YgjfQklaUet-NtNX=H!j&>N zb3A@2J9XuC)C2Xa%Amy|s8o|q0QoTF+CQ^_ckhs^&N#F4x7qFQZ!?wXU2M1w!OuJa zyt(M?O(g!G_n$moUgTRKyE)Vq_po+OdkKh)RdX(yhoj*kBE(#TA9?KgnGn{2XzcA z+(naKu5E68A&_sFyK&_FUX_Zlw70kam4-wj0k{jZ#!A_?cMj3Nx~=X=1g|0Pu#8+K z*#A%gS;)`e%m#Xfr}4nM@0R-cNs-p|$hhg3U`Z>)%t)KV{)i^!HYT>ANlp2_7{n|f z^6XwPRE?rOJ$tqXHb7Lj?sxaDa}nln3UV|@5$eF2K(xpwrU+N!= zbO=DG>c0r*^C%#1F~|_Tcpm4G*F7W*SrE{03re!> z(3!Dx;ot47W!@52J&>20VnH{MPSgv10a-$c8|>lS%t|nXKzpwHRvFKZvIFsSV$|!5 z4=y)7wZYzvHIKzY;0x7vXn5G}YXi~(B%nhHtW%Zm;WXUum8JlD1Ku-E7I4v6o>Bd^ z)AXOeN-Y<`nQ^{!De$R7=ys;=GP%kc-r~lMwfP}z;|y{c=^O=ID$%(?AA%qVaOD76 zpU&|4m%XKof~-aR(YB!F?TV1iXH=hIp0skpqpa$)N;LWvk?xu|6*Xjtf{ZKtk#+ zou-{(axA`1zH1XgWB4;ycu%ELqOR^44vb{B0~8-=5di{kx8$*bV+c4EUHZuD2- zm++U1xsm&p)HYVE$>B%%KTmK8X_6*yNq^gi!h(Q|H8=`M+{A0G$#PAQcU_C|J)?k~ z#qLGr$K3324+rbO1gFEtn@b=x_~Yw8U%&t9StIP*NvQ}ReHM}zKfLiV9yDHN5E9aYZ7xR+H-4z-KiQ39(53cTgZ&hNY9N71YjbifFQ0a@O;@#PUqvEd1 zL|mvfb{1fBEY}xdW$yi|)m~jrLvWEY{PbW=n&bLMU5FEs5#h4&V) zsAL9q6z5vnqBLWhFhb7d`nJ7^lTzx^=KO9ub=~Lv<=dB0FH=)6>u9Qp*wP$DEzGZ z%{mH_Q!4a5TSH!ojIjjymFL8kv@fr1{_7`SFZ8bdMrrbJDs)g6e*5}cG-z0Vq3KV9 z+=09bspcO68dGyIC$-ABhsoKAJnx2rWxx>(kJs54oq3p&;x+dXAx6Kd(TOkbWjuKL z4IH@Hqu2g~YP^Y2m;M;7o1{wB>-U#&ef8FyNufZbz7};7$o{cS#Vin$>H;5=ex^#V znU6XBvI9{)*{oBE5bWgWDEKCyjyFVyS2_Y)nqrPhcRCg8ZvP-fz)?E8^`&jnEN=R< z>Dlx5J%8aoXiMB_w42%RC}`_mSS+Qxt@pi~=9YlhPKBcZ_^QcyuOS~&<=#6c*gSO2 zGIs=U1QA`)GX>W{fb~0J7L&1nks+zSkrzfuXurq1YvA;zZ>eHQ?SVM(<^b(Z)P(~( zpmJzwfKD~DBTK<%s`HNy=41agHjbWysocqZWU(v5p>bnKp+@j}-H2;PV8 zXfb{XW9UJd9)~OyF#K?Gd1s?AaciuRZkmB1D0&NTsAV&z*uvS$lSKW^USTl=hV{&o@x|X^2J}vXVmFjd!e-AS3Z&@L=TxAQ3*PaA)pOqNw z)s@8KfR-fFl8k9gH+S0#WC8f!(GB2Yv_NV7rUN$GKAN%tCjjUx;1I5(ghRI2VF-hS zlm3;{^oKg_Kh^BI!7S*OssE9<(1yq3zmhIVr*zr{ZS5q?GTh0Y#H=5-G}_6Ye5rqe zEEQ2kbYx>@@cBv8aAd}TdY;&FH6q4u%x_rq9AvL}H$noW(y6l>zHxOUv+e+!0YjgL zw?FB!`Nzi6bWh>F18-C-^DjmFr`#7p>DJ9J9$w~p4ml6i+-S2+dh!AyzRfbTI|Pq+tpzh^2OHJfV- zcVqClc<0SJlfl=G6^<&=787}GlR~?B`_&|u7*%z`v&3q(c;~4kjr-Cm>Sn$7D@O}- z9?~nGZ5-1a0pJwTM4ZFh|){zW~WK_-|dBUr!XfTGri5OUK!hRA;LS zMym_?5v2K%a%L(0y_kHMKbym*=G;#VA+$o9uycHvg6f$&FF?{xS=I@gCEu zk=E^B!0hj0YFSoN@wy>kx$H6}>G(M;w7n{ns9{4hYbP=G3*E^Y{OQrQ`+SCP*zH4cCwmq0g z16g1ng}Okzm8w$2z!#-mhF84Y+)U zoKL;@ElKWR9NGcF+rwFo!Dfzke=omsGeM{e4v-21vGl~1n?$EdFp(X0en5k2;A!JA zl2JP(9yqW%9nMpXWuT3_s#y%OmW8%XE}01ILgIwpff=jXYp*TimgBEPUJ2dyEr0!) zFVfu{7TC+!DF;W#R#V{*mwEG#3ts3w$POFoL6~Hk*YuleQ-V#0R&ggWt~4| zW;yCe5NVdj><)UuR?WKNVVwM9dQaqJgVR6oHX@gI&j+a zIk|#xK*i(0>>izOCusX%mO^Qi=QM;g_1Ex%+7sKsC(-Ny5R|{S5~nvn(^mJbj^qh= zq*~%tm+~3h!^J)r+|2K5ThBY)@?oEfhZ-{ZDGz(jx!nsk6DGd855O1tZVc|SIk*v@ zhD5a;^rM}o{n22;3eG<}=MfX2T$Vkk;(&}`Qv@sIW}3&~y-h$Ewlvr*UlBLixt=)D zE>v#_9i)DR9+;^hraQ5PS6^b z8(;!joHq+p`7greholBL)2T=!A{x;(0=4R1^5>H5_S3{l(AGkLEx7253ixL8Iu7AW zLAq~Z8(rSwMcF0000|af@#88x8MqY6>#!S2p!XLqNPIA`s)48uSUW-pCr$G55VTtm zcWFEz374}JRpmtA{^Po-jC(~gD25=m|%be3qjSJ{Bg(Tun>M2?KJ@ zR)h=aC=OO)T3y1-yYP&PhSz<;Bf2Ti)B68*2s3*Etj~wo2%{A{!N3$@iBD8@`7XAQ z(T=JZAkejd4DLbra{4~F3X!f?3NknBmUOIsO?euR(G?45dB=vOtpx3`+Eav@in#attHQt(EE=?vS;>AmU z7aNi4+-LSW9v$#^~{{8oPL%IR%{l8NPf!NJTh>l^*pdwPYf z6-fK$?WVi|v3I)nnz8bL_}D$C02M2gQoue+ zJ$hJ0weJW1YLYkyOAMrPtiku%uF8+&2css`;3LYDLo!=3CkxVyo7_U?_2)C(ru8U7M=05`QdR^< zA-AF)z96XHl@s<%{wFW5eru}1iwHS>h%>?@aPN=%U6bipQm<-gzti1K6=h`dR1-hX5`|b@Shc zAVjP^cPRT5GlT7gP60VQmf;(XsorfIyA56-tuD<-C{HJ9uh|>FPt(@?D}nx;gl}DQ zH)|b|T!6Wg5UG|yP{O?O^dn)|2<_7O)QAjTN56Q85&Go!;HHd@PFZruaIUnlaO7qL z-r@uR20N|1Ti*8II6B84VUCoJF7ANv^8s10kVi)r=;R39XENx_Khp~ovtYCSH6L9` z4(ou->YfuRlLqMXWv!*J^u-Q?H#R9)Z4{=Fz4%O2%7EeORf)VKT+TXGkB8t&#OOuU z9{+Lx4sq|S`yCk!M8TBJ=hQkCPFuA;uGW!5Lo`?bY7>_b7_cJ95PM`1$33w=7poM) z5_(&XV%B%We#?g;K>O^hF`Y8jM~`R}686o~dK7?@Yu&3JZvon}+<(Aj2fq z6Gqf6zj_He?9ir9g;!Rm9DkYYwKK2lva2mo3bTAp6T#f=7?rxEHvSlFmwfdaNrUj2 z$0MiEdwSEsdgo-#&pk)|Z#9kympICRj|rd@tAX2yQw*#-V7+cfRM*5`BF0|x{;t}W zKX2Hwy`Pm)aZ0kvqHdw%>j}NJKAtniBOKq|ixTHwEF&Q7Rq&+Kt28QA zyG$gwtxNCJmScAiDs0+Df6hMhfzfwntPmC7O?#JeGw(cxGGdfGl%eN9O!emUO)Q(J z8Kr2v%pxv`5NoZbY~2&CT_fZByO9PDCJ@R4NdZE6>V<&?%tQGYj{hJc>blSPji@;< z9gD6Yhqr~~Z0ZLj6708TZ^2ZPIK{{517IQ(-k7oO%A%oE8J4T0XRzU-Ma{&jN;f>9 zqFn(dfc7|o8oqN6ZW*VoD(D?=6h@S>A7OWxbo+;*~ z$$#)bNzNmND+tzeDVc7YK{dUF5{4d33dwEQ;6~z3pY!Q4J%JIW z1aBd1Z@%p`O@I~DU8cTb?8#Y9^^H7Sd3`o2eh;=!LqWZWkSEP7U~};mOF7w>H5f`k zy0p5}!KR*;TraRk22;2`dCfZ#joSo#6AuU_yPrncutYnVvLx3yvGYb#m@l8-Z2>Fj zy*B@m=L1t&rmR6nI$4%1GE>?bhVQjgj;f0@DXNa)JW{4=aI_&~btLAG${N_PfolkxYpY7Na~n0`T2wJi35=mb&K z*2SLB6w6e`m_7o0qfcw8Xkz zQYL#sx|6+M$>TQ+apuCTVsnEMRc0hy@8mw^(0RVoy<#-((2U_;NBmoBz&UG;i{hrY z%IHhJBO@CTmnXNS`%cpbg2$_FL8)Cn^(N^b@c!#$$$DM6(l4)HeCzTQp*+q~vu*Le zqzm@}`}|dL>-#)~qy6vb>N<~sA6ogFK`}Kbn#N&Ce933G$FFU_mj!9PfG=rHAEU2oMWT=?nXN4iGmsmpCN1%=bMK1;2LH%Lwq!S?KL@EMOYs>VQnHgkY;@@-;} zB3(e+faSqeA6lu|a}~q|Jf=dX9pWiOt$)yWMk%L|*jZ>+B%%Yo#-8rK90~H9kI<2^ zbXtXTQ`gjYbhOW2{e@zEKF<&S_T8+{PU$Wt zIh}?353$_3E4A%Z%p?i)02f9j|2%nw`}gx}U+P+!9c^pNnlSjn9M}l$uRhq63m;73 zfWs2w;YjTeE*E3{Bcor*@zFC@_Ptj@DzS2@cC(qnJh3p8#rFogJ+~ zcfR5^QfNBMG-!?>@-=5*YEwX}pM2xELicJ?6k!ghZGMkRf^!;9<1Nkr5!+w7RCGz| zbSuBf`ki$U4e5H)N>N5u5CX?rI8VzB&!l7p`C*R?;pTtHKdGz}mv%UexG`)1Dx#+WYwDzn`U!gx*EUI(ZTR!=(M!0yjuc=Y{`f|1W zS64&UJVXu_=ao9+sj21rE>;np_gxpse%XUU)|Z4$!GY|fh~mqP+h98QmA>+}9@Ebn zas}FX0~iU*%Xv0&b#0lhU8!<#Xx}sjbGicF?!W_4f0om~FLPb%2GnBczTaV$%17Vp z>lSw_(*feeZS+@?8-mR~xhywY@&tqz5=S~mR?wHGf$u7Sk&D>yNJXC(Se8wc=#Jt< zwQM`cMI=AV^nL{hBcdvTP`Pp6Cc6s`cGQDv2nG*m%yg1t_K&%o+m7$Oz|^J%8Q8Z0knE`aYoDUR(w&bUleFirq!jb^QA|=T3#&H z3+D;I;vsG@S^IAH0PmfrN4F=Oh3k#y>3ncG5Fhshh)xtA0S+%KY3n^o3u0h_9IzU# zcIcw`P%m3M@KgS7=Qmljm))k-ExTN8L&E5Z77v-T(|;ZmsFgRbghEL(M%(JZF^kbr zAQq49n8!|k#zCzO%4H4?GJd9q(Rsg3t-RM|yYfgBHdJmp_jDcQwU+#x-!Ra&jFLI& zKes#s`Tp})t!*#Sc-7Ly&HS<-RU{{8Z_=9juLF%P4rlBZIt!aY+7DrTtl#Ah?1+s2 zk0m;`Jn4gYCKc+b-u;#jB-0qD%FD~OlcS5#8z^F>bDAuK(*(pd6=E8lJu%xEUN-zfn{JswPfcOnt; z`YaXjGXQzlw2y?xEdf5KtMeDQws*n(Aob28x&Ryio3^DXSq9Pv7=r9*+Y{fMwPMsI z;}09;F)ks1u)x6QOgBG3B%sAWQt#pNj3K8+;Y{K}w#iFfv+AVYXBxNcD$BNoP^gnO zDmN6hD3FzWSrX$MD{|X^DC^)ML~Mpwi4=ab`OB5`({JAVObjAZvSl;UUZMFc{IaCy zi5w%skV1;@j(pJ5@=o94P!b1TMMCk%|7`J4GpAcu%pfz<#|~ zwyh>cX^g9)o8X8TjhqSXwvi3y+#DJ z`dq(Hw;o#%U|79@^_Yd?s~`L9?$r#1L}+tEqsC0pvOfIXj=uR!hrT0&KKYhssxz7{ z4~4`#3@G$$DwpVJj{$tv>utV0l9MJXp!h?6;YeEAOBaqD7fv}m^rA`v_dW|}Io}Wc z;RXaR>*J0GKtNq^>!+p$4u8k`pj?-Mo|_c^;pF2*JcjEN)h{g`9@Xty?qwVHl=nF~ zRPC-;@z_wgO$LKKRjrzPD&aUeU(SKXl+#iswvy-wrBqIVj)w=$u59=eq`mWUP`3Eh zAdc6i1)wAmf(bOM%_pE7t$5W!ffMRpu^U)ONN#XJ{hGM*TJO*0aSHakd1=@<5z3*5 zKPrQlBa({{w5EqdB3EH4UlMAa0 z=Y*$#Mlm)4!q+_Mm!XJ;#A0WAYnCb-l0O2)acGGIKAYc5px5mm-}*UWE4yDMa$w*V zgQ%Y7gz@`CNvvZ~nhgK8adZAFpVh%-(Iy$TtEtB+>I4BHZ}KaV_D|D--lOXI=fgRx z_1gDT7>En2RlJwuo<77zqWT)SI9<4#cMi;rdR<$g&;_1b8i|M9*_34Le%gfdK^7}S)I5YDbtF}D+sUAxZCFJz728j5Avw5U7P+PP^ z0{OJ%23SlVBc)~P7E)cVSO~TJ0(i!wp~2ESsnf3>wrAvgGg#SxTY~_%@q&Gr_D>J1 zj-9o~R|$VC^NyWXkcj(7d2n}l*%h8-DroT|w(6N1ZEmPc0|wd58B#eCR$3fK>nBxPwb}W z`qGuY1E!W`A8%K%C3L!5S?eiBXc;|8l@y^Hmx`cQ)NFhCC-aFly0X*b>;T&!oGxw5 z^EH3MzDov`&zDk_zuPg#r&$VB+nFZ$G;cbHD;5#ni`X!hYVQ~F8QVYk^K($_Vn8H` zT00}#H+gb*Z$n@%bYjKGLlZ>sa{Qs;J>3HjLP&?jbmR_#@|3S?E;mo-G#%bTgL2#E3MWz5Hn`DZ9SD0P zA#pZ!G}GQn=lBgas}$>)kuxkI;5}TvQLBzi)#(v-lbOR%Ziou^8J)Jm#lHs8pWutz zV~wksU$gG4hfr`l&s9XW5BdIV2pn!o;(Fh_hFB{_UxR?!*ppURNDYTZDRqrH^cIBb zuHC(HYB^mu&Qtnp5g*NWo(>t68|2%jV0vh>t?>Bp^jmKZ`9;TZI4e%lK%9kfe17WQN35ZF-pS6?ZSUvzyPgOsGT zLqx#>k$dmclE>OdOXdkYd~;}F=Zn8Zjx^{Ie>qnAD;zhXSUcuPs$nAS0f4flIN$P1 zH&2oNLf3#W{rId|gTK1U{!#Rbv1aID9_A{xsqlugg3r;goNs987ne$f+zj?5y$inL ziHwTra!K6P2l*wJ&51R72QEz8@?L%I5QIy4%ZTa*3IZ}()7K@#uH^(K#S0K)e;HZ? z4Di?}6$+=rnEDa0)6XI~0d-{7?Pbca6Ri}#qM-dKwNyGNzJ#o>HaJIdy1!(!vdtwO z`EL4iwvIPy5RqxweWq@?PYLaPx(1t?cytW5GCQ5@`wKy*#K|0t`q|I%7_ z{7V5QCSJCzUCdzmSw@dOPMqo(KJ9EQZD02tL=FYKzkN-R5(A|`8|77W`95BO0VQ^S zAms;2!M53uB%dF_nYRp~$c0}H86eOTT?ao1Ei`LZuhW7y@9-}^=0=O&_kkGWt~AP! zeR*y{T1DTU>RiWVV4cbm2mE76Ny{aj-fw~ZpQk`wTI#O#S%=cACi~D(Ua6I90IoZ+ zfhU%DD)>stGh>sy3}Q{3E7{ArYneyqkbpe|W{*;UBcgt)TTn*laN^aw^&|!`L`Onm z(-3?W(}}kCp>aH+YU$i7%75&h>S&KQ>M=bcwWU~tHpD_EV!Vz0^>K(9*wHf^VQ*qZ&EPp`S zhLg9u{0G7$k)Gv3q}kAzQNFAAKDuTnKOCVR=1V^-Z&al+v`WeXKN{M9XP48$o*64s zxGABCN`k;Cs>suZ%_GIqI@Ih2j-TKcc~FiHoKY~&z2uoCC3B~ zCCrYTMMMKff%aWQU}sY%^zaEv_WO#LaCp%z%L*A$!6sc2sX z1gL&=T7HkI<;uEB+Q-sbuB2Y}_no@}pJv}$diY!kkh;PnxzR+GUz629yRV{!^_ilsi$iGZj1u0rhO0%n$>{>n%`5TKv2IFlxrB z+|^w=SO$+9DULQ4f=RJ*6a;oTH3~Eei~c-GTGu;8``n=51`nx{$m!ixDsbm-WoV{% z(2Ro}YcN8QO4Rn%m}uG1s!f0mcWh6{LQi=#ppt^)gnKs@X!9?;JTJ0DK@~eFZw5hQ zu4rBRGm$C>Y7GflfhPd&kQ?*+dg1grgb$m?Vt#ozHnG6!lvb6%|}z z8DEAD31p8_a+-B5YLY!GT2pM3N&)RCJC>ryh0-1z9tLg>Rfx1oi_#G^&>11G_U3lX zAF0jY^epz>c#FIJ7zbQFG^y;fRSKhGu0n6^IT*L;l$-7K97)lXJ+Cu()&AQ2Dwkb9 z7~~_e08oZBCp;UtmDzFbvSoO^6;kSiC7eMVWxD@qn&UV%kL-JGUGFF5@0AXdwhFGV{) zNFPhbBPO1$036@xJ@92hlwJcf30TY;bY6P&_@R31n9d* zPB)-TYaxLjp{gOeh}%_9l(kUYk87KX>wKvElK>);oIuX+B9-ZR0e=dF7XmGyws7>Gpi%UoGp+ERc1I}@)@D2R z>A~s-;xc5f1uQhE)sxCmz^@6-cwxcLj;Iq(98->Rmrf}mV2Sajo9_Oa&90y+F42RI zv^OShyDj1hp}gJsk`6T&iY5+Pz^PZ9E)}u=%&{j=bW_!HL`x5Z_9`AoVe9cVV4Fb& zY&;@KNRSqD0rbmGj5Un$p}dWYy>*WZ8VooR&%5oUzJtQ$Ne+~$6Yih26o(G`Ud?j= z*XSbdVxU31IaeNfN`=}99)F8HbDpW7AN-H!RbsiLlYaUW%a!moSsDp82#172q$2t< zhpE2M3o;Y&lY-Z05y;v{eCehGXgDCW(ceUHhSlHZG5jbyjdnT_sTr9m5{WC&vIVEAYYxnG3e4q9r*$VJ!9 zxy%O+>~GW*Xpi2l-n{@CG`= zLqjg*9$jm@_EIkBJfijhSjE8WGTMtW?JLfP-ELUR$iFR4CJ&^VQV}tQG@TM-*H=r) z6(No1pmT!iJ8H|>y}eB#ZOgK7l16hy0XG)fWq9pHNgRf~xI0_v{4IIjM&Vidai*Wm}}Or#6BCxM$GW;E_e*N~l2 z|{m*g2X@;CXzs55AW=4;^Hv>zt?tTK8^%r=bTp=l7FQ?vf!s zB8B!1G^Tx#?#?fuSkpLG>>@Vo`nIH8S2q>j8NOO1oR5zCD>_pmJ+xKkU3*%T2vheT zRtYpzvq7H&iwyB^=c6NTJTNv+wdMZz<|o2FC%;Nu z6wl@cgK0?k9I9DJ*IynXy3Lf}twh!ZhoLsb8z!|4zt?aIVW}snT36DMG1c%!`cqk( z%D_6SaC~@*g6k<9gqWjw6hc&cK3@&O*LzNk@Tz>Fwsr#GZx5Gw?=3jf3*J!nS6jNwhs-#XP132jYKsO0CF)njFE zWWzF|EAis{HOo(uGlC!RGf4mn_K~eZP?+y)LtJ#6QS*@ixk%wLVXejmOvno z<#%=YQ4S9S{(IGOMoo<(uVtYQLM2w0#&&Etz%Qe)P^AvQd}DbUfF>R{Ek@qD0dvpT z__2PQ!1*^K4;z+c@s7xM2PLd<>;d}LFiVLA*e!1qvMI{b-cABB-GQG)L>?lX}b((boePRsHl^5_YJ(TzCh8ick%O}T)NO{-|yy6XC`ll8oaZIm%u z3U#94KRwLu5R0VhNk9&w-$zK_LF^;1g5ldl8-LLosA5A?j{r768ckXdhDa5nhDG{s z`PG}CZ=g1KVU_rU*@R>y2X#IMCkKQ|4g)UhV(u_jUR;3K{S&L&#m2ky~= zNIQ3Z5_TOg#%L0L{T^#v0^0-f5~AROS~B>&Aa+BOKnRHkOCu!&HF150;`twOR16T> zQT+s91jXL^06h=C^uOytrK5 literal 0 HcmV?d00001 diff --git a/docs/source/vectors.rst b/docs/source/vectors.rst new file mode 100644 index 00000000..1a5e8208 --- /dev/null +++ b/docs/source/vectors.rst @@ -0,0 +1,61 @@ +Fragment Network Vectors +======================== + +This document tries to describe how the fragment network vectors are handled. + +Vector Generation +----------------- + +Vector data is generated when a target is uploaded. +The entrypoint is the ``analyse_target()`` function in ``viewer/target_set_upload.py``. +Ultimately it’s a call to ``get_vectors()`` that actually creates the vectors and 3d records. + +There are two relevant database tables, ``hypothesis_vector`` and ``hypothesis_vector3d``. +The first contains the definition of the vectors (e.g. the location of the vector) with respect to the ``viewer_compound`` table. +The second contains the 3D coordinates (start an end point) of the actual 3D vector related to the ``viewer_molecule`` table. + +The ``get_vectors()`` function uses the ``frag.network.decorate.get_3d_vects_for_mol()`` from the +fragalysis repo (https://github.com/xchem/fragalysis) repo to analyse the molecule and create the vector data. +The data is returned as a Python dictionary. For example, for this SMILES ``COc1ccc(Cc2cc(N)nc(N)n2)c(OC)c1OC`` +the following data is generated:: + + { + 'linkers': { + 'COc1ccc([Xe])c(OC)c1OC.Nc1cc([Xe])nc(N)n1': [(-1.3931, -0.3178, 1.6803), (-2.8284, 1.2604, 3.3541), (-3.3955, 0.5713, 2.3605), (-2.8284, 1.2604, 3.3541)] + }, + 'deletions': { + 'COc1ccc(Cc2cc(N)nc([Xe])n2)c(OC)c1OC': [(-2.8284, 1.2604, 3.3541), (-2.8284, 1.2604, 3.3541)], + 'COc1ccc(Cc2cc([Xe])nc(N)n2)c(OC)c1OC': [(-2.7309, -0.2154, 1.5116), (-2.8284, 1.2604, 3.3541)], + 'COc1ccc(Cc2cc(N)nc(N)n2)c([Xe])c1OC': [(-0.7313, 0.3607, 2.6834), (-2.8284, 1.2604, 3.3541)], + 'COc1ccc(Cc2cc(N)nc(N)n2)c(OC)c1[Xe]': [(-1.4984, 1.1631, 3.4943), (-2.8284, 1.2604, 3.3541)], + 'COc1c([Xe])ccc(Cc2cc(N)nc(N)n2)c1OC': [(-0.2665, 0.2739, -1.2642), (-2.8284, 1.2604, 3.3541)] + }, + 'ring': { + 'COc1ccc(C[Xe])c(OC)c1OC.N[Xe].N[Xe]': [(-3.3955, 0.5713, 2.3605), (-2.8284, 1.2604, 3.3541), (-2.8284, 1.2604, 3.3541), (-2.8284, 1.2604, 3.3541), (-2.7309, -0.2154, 1.5116), (-2.8284, 1.2604, 3.3541)], + 'CO[Xe].CO[Xe].CO[Xe].Nc1cc(C[Xe])nc(N)n1': [(-0.7313, 0.3607, 2.6834), (-2.8284, 1.2604, 3.3541), (-1.4984, 1.1631, 3.4943), (-2.8284, 1.2604, 3.3541), (-0.2665, 0.2739, -1.2642), (-2.8284, 1.2604, 3.3541), (-1.3931, -0.3178, 1.6803), (-2.8284, 1.2604, 3.3541)] + }, + 'additions': { + 'COc1ccc(Cc2nc(N)nc(N)c2[Xe])c(OC)c1OC__0': [(-0.7313, 0.3607, 2.6834), (0.3550432234908941, 0.2643353596228167, 2.826830320575495)], + 'COc1cc([Xe])c(Cc2cc(N)nc(N)n2)c(OC)c1OC__0': [(-0.2665, 0.2739, -1.2642), (-1.3590949762544902, 0.3155256056977563, -1.3846305891945447)], + 'COc1c([Xe])cc(Cc2cc(N)nc(N)n2)c(OC)c1OC__0': [(0.5504, 0.9772, -2.1574), (0.09491642463792072, 1.5537821952831012, -2.9759888373645813)] + } + } + +Those molecules are as follows: + +.. image:: vector-mols.png + + +Vector Display +-------------- + +In the Fragalysis front end these vectors are displayed as arrows/cylinders emanating from the appropriate point +in the molecules. These imply a point of change in the molecule e.g. a potential substitution point. + +They are colour coded based on the occurrence of such changes in the fragment network database: +- Red means no examples found +- Yellow means 5 or fewer examples found +- Green means more than 5 examples found +TODO - confirm these numbers. + +TODO - describe how and when that data is generated \ No newline at end of file From 6b8fdb74872e6943fed767f295d973831d3f4c0f Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 1 Dec 2022 10:55:03 +0000 Subject: [PATCH 004/112] Some fixes for configure/ping logic --- design_docs/squonk2-access-control.puml | 73 +++++++++++++++++++++++++ docker-compose.yml | 10 +++- viewer/squonk2_agent.py | 54 ++++++++++++------ 3 files changed, 120 insertions(+), 17 deletions(-) create mode 100644 design_docs/squonk2-access-control.puml diff --git a/design_docs/squonk2-access-control.puml b/design_docs/squonk2-access-control.puml new file mode 100644 index 00000000..b5bf6634 --- /dev/null +++ b/design_docs/squonk2-access-control.puml @@ -0,0 +1,73 @@ +@startuml +' hide the spot +hide circle + +' avoid problems with angled crows feet +skinparam linetype ortho + +entity Squonk2Org #yellowgreen { + *uuid : string[40] + *name : string[80] + *as_url : url + *as_version : string + -- + id : number <> +} + +entity Squonk2Unit #yellowgreen { + *uuid : string[41] + *name : string[80] + -- + id : number <> + project : number <> + organisation : number <> +} + +entity Squonk2Project #yellowgreen { + *uuid : string[44] + *name : string[80] + *product_uuid : string[44] + -- + id : number <> + squonk2_unit_id : number <> + user_id : number <> + target_id : number <> +} + +note right of Squonk2Org + New tables in green +end note + +entity Project { + title : string + -- + id : number <> +} + +note left of Project + The project title + (a hyphenated Proposal/Session string) + is used to create a unit. + Each proposal has its own unit. +end note + +entity User { + -- + *id : number <> +} + +entity SessionProject { + *title : string[200] + *target : number <> + -- + id : number <> +} + +Squonk2Org ||..|{ Squonk2Unit +Squonk2Unit ||..|{ Squonk2Project +Squonk2Unit ||..|| Project +Squonk2Project ||..|| User +Project }o..o{ User +Squonk2Project ||..|| SessionProject + +@enduml diff --git a/docker-compose.yml b/docker-compose.yml index e3fc1cb2..692ed996 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,12 @@ --- + +# You typically create .env file to populate the +# sensitive variables for the stack deployment. +# Then bring the containers up with: - +# docker-compose up -d +# Then enter the stack container with: - +# docker-compose exec stack bash + version: '3' services: @@ -60,7 +68,7 @@ services: OIDC_RP_CLIENT_SECRET: 'c6245428-04c7-466f-9c4f-58c340e981c2' SQUONK2_VERIFY_CERTIFICATES: 'No' SQUONK2_UNIT_BILLING_DAY: 3 - SQUONK2_PRODUCT_FLAVOUR: Bronze + SQUONK2_PRODUCT_FLAVOUR: BRONZE SQUONK2_SLUG: fs-local # The following are normally populated via a local '.env' file # that is not part of the repo, it is a file you create diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 19b2e3bb..b43a690c 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -26,7 +26,7 @@ # Response value for the agent methods Squonk2AgentRv: namedtuple = namedtuple('Squonk2AgentRv', ['success', 'msg']) -SuccessRv: Squonk2AgentRv = Squonk2AgentRv(succes=True, msg=None) +SuccessRv: Squonk2AgentRv = Squonk2AgentRv(success=True, msg=None) # Named tuples are used to pass parameters to the agent methods. # RunJob, used in run_job() @@ -47,8 +47,17 @@ _SUPPORTED_PRODUCT_FLAVOURS: List[str] = ["BRONZE", "SILVER", "GOLD"] +_MAX_SQ2_NAME_LENGTH: int = 80 _MAX_SLUG_LENGTH: int = 10 +_SQ2_NAME_PREFIX: str = "Fragalysis" +# How long are auxiliary parts of Squonk2 names allowed to be? +# This is the maximum name length (assumed to be 80) minus the +# name prefix, minus the slug and minus 2 (for spaces) +_MAX_AUX_STRING_LENGTH: int = _MAX_SQ2_NAME_LENGTH \ + - len(_SQ2_NAME_PREFIX) \ + - _MAX_SLUG_LENGTH \ + - 2 class Squonk2Agent: """Helper class that simplifies access to the Squonk2 Python client. @@ -66,7 +75,12 @@ def __init__(self): # Primary configuration of the module is via the container environment. # We need to recognise that some or all of these may not be defined. # All run-time config that's required is given a __CFG prefix to - # simplify checkign whether all that's required has been defined. + # simplify checking whether all that's required has been defined. + # + # The SQUONK2_SLUG is limited to 10 characters, when combined with + # "Fragalysis {SLUG} ", this leaves (80-22) 58 characters for the + # use with the target-access-string and session project strings + # to form Squonk2 Unit and Project names. self.__CFG_SQUONK2_ASAPI_URL: Optional[str] =\ os.environ.get('SQUONK2_ASAPI_URL') self.__CFG_SQUONK2_DMAPI_URL: Optional[str] =\ @@ -80,7 +94,7 @@ def __init__(self): self.__CFG_SQUONK2_PRODUCT_FLAVOUR: Optional[str] =\ os.environ.get('SQUONK2_PRODUCT_FLAVOUR') self.__CFG_SQUONK2_SLUG: Optional[str] =\ - os.environ.get('SQUONK2_SLUG') + os.environ.get('SQUONK2_SLUG')[:_MAX_SLUG_LENGTH] self.__CFG_SQUONK2_ORG_OWNER: Optional[str] =\ os.environ.get('SQUONK2_ORG_OWNER') self.__CFG_SQUONK2_ORG_OWNER_PASSWORD: Optional[str] =\ @@ -121,6 +135,8 @@ def __init__(self): self.__org_record: Optional[Squonk2Org] = None self.__owner_token: str = '' + self.__keycloak_hostname: str = '' + self.__keycloak_realm: str = '' def _get_org_owner_token(self) -> Optional[str]: """Gets an access token for the Squonk2 organisation owner. @@ -201,17 +217,23 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # Organisation is known to AS, and it hasn't changed. return SuccessRv - def _ensure_unit(self, proposal: str) -> Squonk2AgentRv: - """Gets or creates a Squonk2 Unit. + def _ensure_unit(self, target_access_string: str) -> Squonk2AgentRv: + """Gets or creates a Squonk2 Unit based on a customer's "target access string" + (TAS). If a Unit is created its name will begin with the text "Fragalysis " + followed by the configured 'SQUONK2_SLUG' (chosen to be unique between all + Fragalysis instances that share the same Squonk2 service) and then the + TAS. In DLS the TAS is essentially the "proposal". On success the returned message is used to carry the Squonk2 project UUID. """ - assert proposal + assert project assert self.__org_record - unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(proposal=proposal).first() + unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(proposal=project).first() if not unit: - unit_name: styr = f'Fragalysis {self.__CFG_SQUONK2_SLUG} {proposal}' + unit_name: styr = 'Fragalysis' \ + f' {self.__CFG_SQUONK2_SLUG}' \ + f' {target_access_string[:_MAX_AUX_STRING_LENGTH]}' rv: AsApiRv = AsApi.create_unit(unit_name=unit_name, org_id=self.__org_record.uuid, billing_day=self.__unit_billing_day) @@ -281,7 +303,6 @@ def _ensure_project(self, return Squonk2AgentRv(success=True, msg=project.uuid) - @property def configured(self) -> Squonk2AgentRv: """Returns True if the module appears to be configured, i.e. all the environment variables appear to be set. @@ -290,7 +311,7 @@ def configured(self) -> Squonk2AgentRv: # static (environment) variables, if we've been here before # just return our previous result. if self.__configuration_checked: - return self.__configured, None + return Squonk2AgentRv(success=self.__configured, msg=None) self.__configuration_checked = True for name, value in self.__dict__.items(): @@ -325,7 +346,7 @@ def configured(self) -> Squonk2AgentRv: ' but the value is not a number'\ f' ({ self.__CFG_SQUONK2_UNIT_BILLING_DAY})' _LOGGER.error(msg) - return False, msg + return Squonk2AgentRv(success=False, msg=msg) self.__unit_billing_day = int(self.__CFG_SQUONK2_UNIT_BILLING_DAY) if self.__unit_billing_day < 1: msg = 'SQUONK2_UNIT_BILLING_DAY cannot be less than 1' @@ -334,7 +355,8 @@ def configured(self) -> Squonk2AgentRv: # Product tier to upper-case if not self.__CFG_SQUONK2_PRODUCT_FLAVOUR in _SUPPORTED_PRODUCT_FLAVOURS: - msg = 'SQUONK2_PRODUCT_FLAVOUR is not supported' + msg = f'SQUONK2_PRODUCT_FLAVOUR ({self.__CFG_SQUONK2_PRODUCT_FLAVOUR})' \ + ' is not supported' _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) self.__product_flavour = self.__CFG_SQUONK2_PRODUCT_FLAVOUR.upper() @@ -350,7 +372,6 @@ def configured(self) -> Squonk2AgentRv: return SuccessRv - @property def ping(self) -> Squonk2AgentRv: """Returns True if all the Squonk2 installations referred to by the URLs respond. @@ -359,7 +380,7 @@ def ping(self) -> Squonk2AgentRv: by calling on '_pre_flight_checks()'. If the org is known a Squonk2Org record is created, if not the ping fails. """ - if not self.configured: + if not self.configured(): msg: str = 'Not configured' _LOGGER.debug(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -382,7 +403,8 @@ def ping(self) -> Squonk2AgentRv: url = f'{self.__CFG_SQUONK2_DMAPI_URL}/api' try: resp = requests.head(url, verify=self.__verify_certificates) - except: + except Exception as ex: + print(ex) _LOGGER.warning('Exception checking DM at %s', url) if resp is None or resp.status_code != 308: msg = f'Data Manager is not responding from {url}' @@ -427,7 +449,7 @@ def run_job(self, params: RunJob) -> Squonk2AgentRv: return SuccessRv def send(self, params: Send) -> Squonk2AgentRv: - """A blocking method that takes care of send a set of files to + """A blocking method that takes care of sending a set of files to the configured Squonk2 installation. """ assert params From 222f59e4ab32086f86112cfd93112818baf1cb40 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 1 Dec 2022 15:47:18 +0000 Subject: [PATCH 005/112] More Squonk Agent bugfixes configure() and ping() appear to work --- viewer/squonk2_agent.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index b43a690c..d59a7118 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -151,7 +151,7 @@ def _get_org_owner_token(self) -> Optional[str]: password=self.__CFG_SQUONK2_ORG_OWNER_PASSWORD, ) if not self.__owner_token: - _LOGGER.warning('Failed to get access token for Squonk2 org owner') + _LOGGER.warning('Failed to get access token for AS Organisation owner') return None # OK if we get here return self.__owner_token @@ -160,16 +160,12 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: """Execute pre-flight checks, can be called multiple times, it acts only once. """ - # Been here before, and successful? - # If not try the pre-flight check again. - if self.__pre_flight_check_status: - return Squonk2AgentRv(success=True, msg=None) # If a Squonk2Org record exists its UUID cannot have changed. # We cannot change the organisation once deployed. The corresponding Units, # Products and Projects are organisation-specific. The Squonk2Org table # records the organisation ID and the Account Server URL where the ID - # is valid. Neither of these values can change once deployed. + # is valid. None of these values can change once deployed. squonk2_org: Optional[Squonk2Org] = Squonk2Org.objects.all().first() if squonk2_org and squonk2_org.uuid != self.__CFG_SQUONK2_ORG_UUID: @@ -186,27 +182,25 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # Get the ORG from the AS API. # If it knows the org the response will be successful, # and we'll also have the Org's name. - as_rv = AsApi.get_organisation(self.__owner_token, - org_id=self.__CFG_SQUONK2_ORG_UUID) - if not as_rv.success: - msg = 'Failed to get Organisation from Account Server' - print(msg) - return quonk2AgentRv(success=False, msg=msg) + as_o_rv = AsApi.get_organisation(self.__owner_token, + org_id=self.__CFG_SQUONK2_ORG_UUID) + if not as_o_rv.success: + msg = 'Failed checking AS Organisation' + return Squonk2AgentRv(success=False, msg=msg) # The org is known to the AS. # Get the AS API version (for reference) - as_rv: AsApiRv = AsApi.get_version() - if not as_rv.success: - msg = 'Failed to get version from Account Server' - print(msg) - return quonk2AgentRv(success=False, msg=msg) - as_version: str = as_rv.msg['version'] + as_v_rv: AsApiRv = AsApi.get_version() + if not as_v_rv.success: + msg = 'Failed to get version from AS' + return Squonk2AgentRv(success=False, msg=msg) + as_version: str = as_v_rv.msg['version'] # If there's no Squonk2Org record, create one, # recording the ORG ID and the AS we used to verify it exists. if not squonk2_org: squonk2_org = Squonk2Org(uuid=self.__CFG_SQUONK2_ORG_UUID, - name=as_rv.msg['name'], + name=as_o_rv.msg['name'], as_url=self.__CFG_SQUONK2_ASAPI_URL, as_version=as_version) squonk2_org.save() From 4d4f46a573182e4863ee57ff862e53f7758c7744 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 1 Dec 2022 15:48:07 +0000 Subject: [PATCH 006/112] Removed rogue calls to print() --- viewer/squonk2_agent.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index d59a7118..8c794378 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -389,7 +389,6 @@ def ping(self) -> Squonk2AgentRv: _LOGGER.warning('Exception checking UI at %s', url) if resp is None or resp.status_code != 200: msg = f'UI is not responding from {url}' - print(msg) _LOGGER.debug(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -398,11 +397,9 @@ def ping(self) -> Squonk2AgentRv: try: resp = requests.head(url, verify=self.__verify_certificates) except Exception as ex: - print(ex) _LOGGER.warning('Exception checking DM at %s', url) if resp is None or resp.status_code != 308: msg = f'Data Manager is not responding from {url}' - print(msg) _LOGGER.debug(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -414,7 +411,6 @@ def ping(self) -> Squonk2AgentRv: _LOGGER.warning('Exception checking AS at %s', url) if resp is None or resp.status_code != 308: msg = f'Account Manager is not responding from {url}' - print(msg) _LOGGER.debug(msg) return Squonk2AgentRv(success=False, msg=msg) From 2b61bccbca7dba4c46be20b02c113e6fed122fb7 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 1 Dec 2022 15:56:55 +0000 Subject: [PATCH 007/112] Adds some info logging to Squonk2Agent --- viewer/squonk2_agent.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 8c794378..074749f1 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -143,11 +143,18 @@ def _get_org_owner_token(self) -> Optional[str]: This sets the __keycloak_hostname member and also returns the token. """ assert self.__keycloak_hostname + + _LOGGER.info('__keycloak_hostname="%s" __keycloak_realm="%s" client=%s org_owner=%s', + self.__keycloak_hostname, + self.__keycloak_realm, + self.__CFG_OIDC_AS_CLIENT_ID, + self.__CFG_OIDC_AS_CLIENT_ID) + self.__owner_token = Auth.get_access_token( keycloak_url="https://" + self.__keycloak_hostname + "/auth", keycloak_realm=self.__keycloak_realm, keycloak_client_id=self.__CFG_OIDC_AS_CLIENT_ID, - username=self.__CFG_SQUONK2_ORG_OWNER, + username=self.self.__CFG_OIDC_AS_CLIENT_ID, password=self.__CFG_SQUONK2_ORG_OWNER_PASSWORD, ) if not self.__owner_token: @@ -182,6 +189,7 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # Get the ORG from the AS API. # If it knows the org the response will be successful, # and we'll also have the Org's name. + _LOGGER.info('Checking organisation (%s)', self.__CFG_SQUONK2_ORG_UUID) as_o_rv = AsApi.get_organisation(self.__owner_token, org_id=self.__CFG_SQUONK2_ORG_UUID) if not as_o_rv.success: @@ -431,7 +439,7 @@ def run_job(self, params: RunJob) -> Squonk2AgentRv: assert isinstance(params, RunJob) # Protect against lack of config or connection/setup issues... - if not self.ping: + if not self.ping(): msg: str = 'Squonk2 ping failed.'\ ' Are we configured properly and is Squonk alive?' return Squonk2AgentRv(success=False, msg=msg) @@ -446,7 +454,7 @@ def send(self, params: Send) -> Squonk2AgentRv: assert isinstance(params, Send) # Protect against lack of config or connection/setup issues... - if not self.ping: + if not self.ping(): msg: str = 'Squonk2 ping failed.'\ ' Are we configured properly and is Squonk alive?' return Squonk2AgentRv(success=False, msg=msg) From bef48e6b452d148ce65a897bcf9e46b3923e17ac Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 6 Dec 2022 16:18:42 +0000 Subject: [PATCH 008/112] Some fixes for AC --- docker-compose.yml | 1 + viewer/squonk2_agent.py | 67 ++++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 692ed996..2dc763b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,6 +75,7 @@ services: # and is used by docker-compose if it finds it. OIDC_KEYCLOAK_REALM: ${OIDC_KEYCLOAK_REALM} OIDC_AS_CLIENT_ID: ${OIDC_AS_CLIENT_ID} + OIDC_DM_CLIENT_ID: ${OIDC_DM_CLIENT_ID} SQUONK2_ORG_OWNER: ${SQUONK2_ORG_OWNER} SQUONK2_ORG_OWNER_PASSWORD: ${SQUONK2_ORG_OWNER_PASSWORD} SQUONK2_ORG_UUID: ${SQUONK2_ORG_UUID} diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 074749f1..aaf06ed8 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -38,10 +38,15 @@ "callback_url"]) # Send, used in send() -Send: namedtuple = namedtuple("Send", ["access_token", - "proposal", +# Access ID is the Fragalysis Project record ID +# Session ID is the Fragalysis SessionProject record ID +# Target ID is the Fragalysis Target record ID +# Snapshot ID is the Fragalysis Snapshot record ID +Send: namedtuple = namedtuple("Send", ["token", "user_id", + "access_id", "target_id", + "session_id", "snapshot_id"]) @@ -59,6 +64,10 @@ - _MAX_SLUG_LENGTH \ - 2 +# True if the code's in Test Mode +_TEST_MODE: bool = True + + class Squonk2Agent: """Helper class that simplifies access to the Squonk2 Python client. Users shouldn't instantiate the class directly, instead they should @@ -101,6 +110,8 @@ def __init__(self): os.environ.get('SQUONK2_ORG_OWNER_PASSWORD') self.__CFG_OIDC_AS_CLIENT_ID: Optional[str] = \ os.environ.get('OIDC_AS_CLIENT_ID') + self.__CFG_OIDC_DM_CLIENT_ID: Optional[str] = \ + os.environ.get('OIDC_DM_CLIENT_ID') self.__CFG_OIDC_KEYCLOAK_REALM: Optional[str] = \ os.environ.get('OIDC_KEYCLOAK_REALM') @@ -154,7 +165,7 @@ def _get_org_owner_token(self) -> Optional[str]: keycloak_url="https://" + self.__keycloak_hostname + "/auth", keycloak_realm=self.__keycloak_realm, keycloak_client_id=self.__CFG_OIDC_AS_CLIENT_ID, - username=self.self.__CFG_OIDC_AS_CLIENT_ID, + username=self.__CFG_SQUONK2_ORG_OWNER, password=self.__CFG_SQUONK2_ORG_OWNER_PASSWORD, ) if not self.__owner_token: @@ -219,7 +230,7 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # Organisation is known to AS, and it hasn't changed. return SuccessRv - def _ensure_unit(self, target_access_string: str) -> Squonk2AgentRv: + def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: """Gets or creates a Squonk2 Unit based on a customer's "target access string" (TAS). If a Unit is created its name will begin with the text "Fragalysis " followed by the configured 'SQUONK2_SLUG' (chosen to be unique between all @@ -228,11 +239,19 @@ def _ensure_unit(self, target_access_string: str) -> Squonk2AgentRv: On success the returned message is used to carry the Squonk2 project UUID. """ - assert project assert self.__org_record + if not _TEST_MODE: + assert access_id + + # Get the target access string + target_access_string: str = '' + if _TEST_MODE: + _LOGGER.warning('Using FALLBACK_PROPOSAL_ID') + target_access_string = self.__FALLBACK_PROPOSAL_ID - unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(proposal=project).first() + unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(proposal=target_access_string).first() if not unit: + _LOGGER.info('No existing Unit for this TAS (%s)', target_access_string) unit_name: styr = 'Fragalysis' \ f' {self.__CFG_SQUONK2_SLUG}' \ f' {target_access_string[:_MAX_AUX_STRING_LENGTH]}' @@ -241,20 +260,25 @@ def _ensure_unit(self, target_access_string: str) -> Squonk2AgentRv: billing_day=self.__unit_billing_day) if not rv.success: msg: str = rv.msg['error'] + _LOGGER.error('Failed to create Unit for TAS (%s)', target_access_string) return Squonk2AgentRv(success=False, msg=msg) + unit_uuid: str = rv.msg['id'] unit: Squonk2Unit = Squonk2Unit(uuid=unit_uuid, name=unit_name, - proposal=proposal, + proposal=target_access_string, organisation=self.__org_record.id) unit.save() + else: + _LOGGER.debug('Unit %s already exists for this TAS (%s)', + unit.uuid, + target_access_string) return Squonk2AgentRv(success=True, msg=unit.uuid) def _ensure_project(self, - user_id: int, target_id: int, - proposal: str) -> Squonk2AgentRv: + user_id: int) -> Squonk2AgentRv: """Gets or creates a Squonk2 Project, used as the destination of files and job executions. Each project requires an AS Product (tied to the User and Target) and Unit (tied to the proposal). @@ -264,20 +288,22 @@ def _ensure_project(self, given has been checked. On success the returned message is used to carry the Squonk2 project UUID. + + For testing the target and user IDs are permitted to be 0. """ - assert user_id - assert user_id > 0 - assert target_id - assert target_id > 0 - assert proposal + if not _TEST_MODE: + assert target_id > 0 + assert user_id > 0 - # A Squonk2Unit must exist for the Proposal, and there must be + # A Squonk2Unit must exist for the Target Access String, and there must be # a Squonk2Project record for the user/target combination. - # If not it is created. - rv: Squonk2AgentRv = self._ensure_unit(proposal) + # If not they are created. + + rv: Squonk2AgentRv = self._ensure_unit(target_id) if not rv.success: return rv unit_uuid: str = rv.msg + print(f'Got or Created Unit {unit_uuid}') # A Squonk2Project record must exist for the unit/user/target combination. # If not it is created. @@ -459,6 +485,13 @@ def send(self, params: Send) -> Squonk2AgentRv: ' Are we configured properly and is Squonk alive?' return Squonk2AgentRv(success=False, msg=msg) + rv_u: Squonk2AgentRv = self._ensure_project(params.user_id, + params.access_id, + params.target_id) + if not rv_u.succes: + msg: str = 'Failed to create corresponding Squonk2 Project' + return Squonk2AgentRv(success=False, msg=msg) + return SuccessRv From 9515b85b347441a6da6c7dfa3e2029a155134130 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 7 Dec 2022 17:35:16 +0000 Subject: [PATCH 009/112] More work on Squonk2 Agent - Now creates units, products and projects - Adds user as editor of project - Use of latest client library --- docker-compose.yml | 5 +- requirements.txt | 2 +- viewer/migrations/0025_squonk2org.py | 23 -- ...5_squonk2org_squonk2project_squonk2unit.py | 45 +++ viewer/models.py | 18 +- viewer/squonk2_agent.py | 350 +++++++++++++----- 6 files changed, 314 insertions(+), 129 deletions(-) delete mode 100644 viewer/migrations/0025_squonk2org.py create mode 100644 viewer/migrations/0025_squonk2org_squonk2project_squonk2unit.py diff --git a/docker-compose.yml b/docker-compose.yml index 2dc763b1..ae4d0eca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: image: postgres:12.2 container_name: database volumes: - - ../data/postgre/data:/var/lib/postgresql/data + - ../data/postgresql/data:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: fragalysis POSTGRES_USER: fragalysis @@ -82,6 +82,9 @@ services: SQUONK2_UI_URL: ${SQUONK2_UI_URL} SQUONK2_DMAPI_URL: ${SQUONK2_DMAPI_URL} SQUONK2_ASAPI_URL: ${SQUONK2_ASAPI_URL} + FALLBACK_TAS: ${FALLBACK_TAS} + DUMMY_USER: ${DUMMY_USER} + DUMMY_SESSION_TITLE: ${DUMMY_SESSION_TITLE} ports: - "8080:80" depends_on: diff --git a/requirements.txt b/requirements.txt index 2809b491..58b164bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.12.0 +im-squonk2-client==1.14.0 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 diff --git a/viewer/migrations/0025_squonk2org.py b/viewer/migrations/0025_squonk2org.py deleted file mode 100644 index 209f55d8..00000000 --- a/viewer/migrations/0025_squonk2org.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 3.1.14 on 2022-10-19 09:24 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('viewer', '0024_add_job_request_start_and_finish_times'), - ] - - operations = [ - migrations.CreateModel( - name='Squonk2Org', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.TextField(max_length=40)), - ('name', models.TextField(max_length=80)), - ('as_url', models.URLField()), - ('as_version', models.TextField()), - ], - ), - ] diff --git a/viewer/migrations/0025_squonk2org_squonk2project_squonk2unit.py b/viewer/migrations/0025_squonk2org_squonk2project_squonk2unit.py new file mode 100644 index 00000000..43bb4738 --- /dev/null +++ b/viewer/migrations/0025_squonk2org_squonk2project_squonk2unit.py @@ -0,0 +1,45 @@ +# Generated by Django 3.1.14 on 2022-12-07 14:21 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('viewer', '0024_add_job_request_start_and_finish_times'), + ] + + operations = [ + migrations.CreateModel( + name='Squonk2Org', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.TextField(max_length=40)), + ('name', models.TextField(max_length=80)), + ('as_url', models.URLField()), + ('as_version', models.TextField()), + ], + ), + migrations.CreateModel( + name='Squonk2Unit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.TextField(max_length=41)), + ('name', models.TextField()), + ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='viewer.squonk2org')), + ], + ), + migrations.CreateModel( + name='Squonk2Project', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.TextField(max_length=44)), + ('name', models.TextField()), + ('product_uuid', models.TextField(max_length=44)), + ('unit', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='viewer.squonk2unit')), + ], + ), + ] diff --git a/viewer/models.py b/viewer/models.py index 24d7ff96..30c5b3c2 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -1236,16 +1236,18 @@ class Squonk2Unit(models.Model): consisting of 'unit-' followed by a uuid4 value, e.g. 'unit-54260047-183b-42e8-9658-385a1e1bd236' name: TextField (80) - The name of the Squonk2 Unit UUID (obtained form the AS). - project: ForeignKey - A Foreign Key to the Project (Proposal) the Unit belongs to. + The name used to create the Squonk2 Unit UUID + This is not limited by the actual name length imposed by the DM + target_access: ForeignKey + A Foreign Key to the Project (Proposal) the Unit belongs to, + a record that contains the "target access string". organisation: ForeignKey A Foreign Key to the Organisation the Unit belongs to. """ uuid = models.TextField(max_length=41, null=False) - name = models.TextField(max_length=80, null=False) + name = models.TextField(null=False) - project = models.ForeignKey(Project, null=False, on_delete=models.CASCADE) +# target_access = models.ForeignKey(Project, null=False, on_delete=models.CASCADE) organisation = models.ForeignKey(Squonk2Org, null=False, on_delete=models.CASCADE) class Squonk2Project(models.Model): @@ -1265,11 +1267,11 @@ class Squonk2Project(models.Model): e.g. 'product-54260047-183b-42e8-9658-385a1e1bd236' """ uuid = models.TextField(max_length=44, null=False) - name = models.TextField(max_length=80, null=False) + name = models.TextField(null=False) product_uuid = models.TextField(max_length=44, null=False) unit = models.ForeignKey(Squonk2Unit, null=False, on_delete=models.CASCADE) - user = models.ForeignKey(User, null=False, on_delete=models.CASCADE) - target = models.ForeignKey(Target, null=False, on_delete=models.CASCADE) +# user = models.ForeignKey(User, null=False, on_delete=models.CASCADE) +# session_project = models.ForeignKey(SessionProject, null=False, on_delete=models.CASCADE) # End of Squonk Job Tables diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index aaf06ed8..25e180e0 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -16,10 +16,12 @@ from django.core.exceptions import ObjectDoesNotExist from squonk2.auth import Auth from squonk2.as_api import AsApi, AsApiRv +from squonk2.dm_api import DmApi, DmApiRv import requests from requests import Response from wrapt import synchronized +from viewer.models import User, SessionProject, Project from viewer.models import Squonk2Project, Squonk2Org, Squonk2Unit _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -28,26 +30,27 @@ Squonk2AgentRv: namedtuple = namedtuple('Squonk2AgentRv', ['success', 'msg']) SuccessRv: Squonk2AgentRv = Squonk2AgentRv(success=True, msg=None) +# Parameters common to each named Tuple +CommonParams: namedtuple = namedtuple("CommonParams", + ["token", + "user_id", + "access_id", + "target_id", + "session_id"]) + # Named tuples are used to pass parameters to the agent methods. # RunJob, used in run_job() -RunJob: namedtuple = namedtuple("RunJob", ["access_token", - "proposal", - "user_id", - "target_id", - "job_spec", - "callback_url"]) +RunJobParams: namedtuple = namedtuple("RunJob", ["common", + "job_spec", + "callback_url"]) # Send, used in send() # Access ID is the Fragalysis Project record ID # Session ID is the Fragalysis SessionProject record ID # Target ID is the Fragalysis Target record ID # Snapshot ID is the Fragalysis Snapshot record ID -Send: namedtuple = namedtuple("Send", ["token", - "user_id", - "access_id", - "target_id", - "session_id", - "snapshot_id"]) +SendParams: namedtuple = namedtuple("Send", ["common", + "snapshot_id"]) _SUPPORTED_PRODUCT_FLAVOURS: List[str] = ["BRONZE", "SILVER", "GOLD"] @@ -56,13 +59,11 @@ _MAX_SLUG_LENGTH: int = 10 _SQ2_NAME_PREFIX: str = "Fragalysis" -# How long are auxiliary parts of Squonk2 names allowed to be? -# This is the maximum name length (assumed to be 80) minus the -# name prefix, minus the slug and minus 2 (for spaces) -_MAX_AUX_STRING_LENGTH: int = _MAX_SQ2_NAME_LENGTH \ - - len(_SQ2_NAME_PREFIX) \ - - _MAX_SLUG_LENGTH \ - - 2 +_SQ2_PRODUCT_TYPE: str = 'DATA_MANAGER_PROJECT_TIER_SUBSCRIPTION' +_SQ2_PRODUCT_FLAVOUR: str = 'GOLD' + +# How long are Squonk2 'names'? +_SQ2_MAX_NAME_LENGTH: int = 80 # True if the code's in Test Mode _TEST_MODE: bool = True @@ -116,10 +117,14 @@ def __init__(self): os.environ.get('OIDC_KEYCLOAK_REALM') # Optional config (no '__CFG_' prefix) - self.__FALLBACK_PROPOSAL_ID: Optional[str] =\ - os.environ.get('FALLBACK_PROPOSAL_ID') - self.__FORCE_FALLBACK_PROPOSAL_ID: Optional[str] =\ - os.environ.get('FORCE_FALLBACK_PROPOSAL_ID') + self.__DUMMY_SESSION_TITLE: Optional[str] =\ + os.environ.get('DUMMY_SESSION_TITLE') + self.__DUMMY_USER: Optional[str] =\ + os.environ.get('DUMMY_USER') + self.__FALLBACK_TAS: Optional[str] =\ + os.environ.get('FALLBACK_TAS') + self.__FORCE_FALLBACK_TAS: Optional[str] =\ + os.environ.get('FORCE_FALLBACK_TAS') self.__SQUONK2_VERIFY_CERTIFICATES: Optional[str] = \ os.environ.get('SQUONK2_VERIFY_CERTIFICATES') @@ -145,34 +150,81 @@ def __init__(self): # Set on successful 'pre-flight-check' self.__org_record: Optional[Squonk2Org] = None - self.__owner_token: str = '' + self.__org_owner_as_token: str = '' + self.__org_owner_dm_token: str = '' self.__keycloak_hostname: str = '' self.__keycloak_realm: str = '' - def _get_org_owner_token(self) -> Optional[str]: - """Gets an access token for the Squonk2 organisation owner. - This sets the __keycloak_hostname member and also returns the token. + def _build_unit_name(self, target_access_string: str) -> Tuple[str, str]: + assert target_access_string + # AS Units are named using the Target Access String (TAS) + # which, in Diamond will be a "visit" string like "lb00000-1" + name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} /{target_access_string}/' + return name[:_SQ2_MAX_NAME_LENGTH], name + + def _build_product_name(self, username: str, session_string: str) -> Tuple[str, str]: + """Builds a Product name, returning the truncated and un-truncated form""" + assert username + assert session_string + # AS Products (there's a 1:1 mapping to DM Projects) + # are named using the user and the session + + # The Product name characters are not restricted + identifier: str = f'{username}::{session_string}' + name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} {identifier}' + return name[:_SQ2_MAX_NAME_LENGTH], name + + def _build_project_name(self, user_id: int, session_id: int) -> Tuple[str, str]: + assert user_id + assert session_id + # DM Projects (there's a 1:1 mapping to Products) + # are named using the user and the session + + # The Project name characters are RESTRICTED, + # and need to be limited to characters that are + # valid for use with RFC 1123 Label Names + identifier: str = f'{user_id}-{session_id}' + name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} {identifier}' + return name[:_SQ2_MAX_NAME_LENGTH], name + + def _get_squonk2_owner_tokens(self) -> Optional[Tuple[str, str]]: + """Gets access tokens for the Squonk2 organisation owner. + This sets the __keycloak_hostname member and also returns the tokens, + getting one for the AS and one for the DM. """ assert self.__keycloak_hostname - _LOGGER.info('__keycloak_hostname="%s" __keycloak_realm="%s" client=%s org_owner=%s', + _LOGGER.info('__keycloak_hostname="%s" __keycloak_realm="%s" dm-client=%s as-client=%s org_owner=%s', self.__keycloak_hostname, self.__keycloak_realm, + self.__CFG_OIDC_DM_CLIENT_ID, self.__CFG_OIDC_AS_CLIENT_ID, self.__CFG_OIDC_AS_CLIENT_ID) - self.__owner_token = Auth.get_access_token( + self.__org_owner_as_token = Auth.get_access_token( keycloak_url="https://" + self.__keycloak_hostname + "/auth", keycloak_realm=self.__keycloak_realm, keycloak_client_id=self.__CFG_OIDC_AS_CLIENT_ID, username=self.__CFG_SQUONK2_ORG_OWNER, password=self.__CFG_SQUONK2_ORG_OWNER_PASSWORD, ) - if not self.__owner_token: + if not self.__org_owner_as_token: _LOGGER.warning('Failed to get access token for AS Organisation owner') return None + + self.__org_owner_dm_token = Auth.get_access_token( + keycloak_url="https://" + self.__keycloak_hostname + "/auth", + keycloak_realm=self.__keycloak_realm, + keycloak_client_id=self.__CFG_OIDC_DM_CLIENT_ID, + username=self.__CFG_SQUONK2_ORG_OWNER, + password=self.__CFG_SQUONK2_ORG_OWNER_PASSWORD, + ) + if not self.__org_owner_dm_token: + _LOGGER.warning('Failed to get access token for DM as AS Organisation owner') + return None + # OK if we get here - return self.__owner_token + return self.__org_owner_as_token, self.__org_owner_dm_token def _pre_flight_checks(self) -> Squonk2AgentRv: """Execute pre-flight checks, @@ -193,15 +245,15 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # OK, so the ORG UUID has not changed. # Is it known to the configured AS? - if not self._get_org_owner_token(): - msg = 'Failed to get AS token for organisation owner' + if not self._get_squonk2_owner_tokens(): + msg = 'Failed to get AS or DM token for organisation owner' return Squonk2AgentRv(success=False, msg=msg) # Get the ORG from the AS API. # If it knows the org the response will be successful, # and we'll also have the Org's name. _LOGGER.info('Checking organisation (%s)', self.__CFG_SQUONK2_ORG_UUID) - as_o_rv = AsApi.get_organisation(self.__owner_token, + as_o_rv = AsApi.get_organisation(self.__org_owner_as_token, org_id=self.__CFG_SQUONK2_ORG_UUID) if not as_o_rv.success: msg = 'Failed checking AS Organisation' @@ -230,6 +282,77 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # Organisation is known to AS, and it hasn't changed. return SuccessRv + def _create_product_and_project(self, + unit: Squonk2Unit, + user_name: str, + session_title: str, + params: CommonParams) -> Squonk2AgentRv: + """Called if a Product (and Project) needs to be created. + """ + assert unit + assert user_name + assert session_title + assert params + + # Create an AS Product. + name_truncated, name_full = self._build_product_name(user_name, session_title) + msg: str = f'Creating AS Product "{name_full}"...' + print(msg) + _LOGGER.info(msg) + + as_rv: AsApiRv = AsApi.create_product(self.__org_owner_as_token, + product_name=name_truncated, + unit_id=unit.uuid, + product_type=_SQ2_PRODUCT_TYPE, + flavour=_SQ2_PRODUCT_FLAVOUR) + print(as_rv) + assert as_rv.success + product_uuid: str = as_rv.msg['id'] + msg = f'Created AS Product "{product_uuid}"...' + print(msg) + _LOGGER.info(msg) + + # Create a DM Project + name_truncated, name_full = self._build_project_name(1, 2) + msg = f'Creating DM Project "{name_full}"...' + print(msg) + _LOGGER.info(msg) + + dm_rv: DmApiRv = DmApi.create_project(self.__org_owner_dm_token, + project_name=name_truncated, + as_tier_product_id=product_uuid) + print(dm_rv) + assert dm_rv.success + project_uuid: str = dm_rv.msg["project_id"] + msg = f'Created DM Project "{project_uuid}"...' + print(msg) + _LOGGER.info(msg) + + # Add the user as an Editor to the Project + msg = f'Adding "{user_name} to DM Project as Editor...' + print(msg) + _LOGGER.info(msg) + dm_rv = DmApi.add_project_editor(self.__org_owner_dm_token, + project_id=project_uuid, + editor=user_name) + print(dm_rv) + assert dm_rv.success + msg = f'Added "{user_name} to DM Project as Editor' + print(msg) + _LOGGER.info(msg) + + # Now record these remote objects in a new + # Squonk2Project record... + sq2_project = Squonk2Project(uuid=project_uuid, + name=name_full, + product_uuid=product_uuid, + unit_id=unit.id) + sq2_project.save() + + # If the second call fails - delete the object created in the first + + return Squonk2AgentRv(success=True, msg=sq2_project) + def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: """Gets or creates a Squonk2 Unit based on a customer's "target access string" (TAS). If a Unit is created its name will begin with the text "Fragalysis " @@ -243,19 +366,27 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: if not _TEST_MODE: assert access_id - # Get the target access string - target_access_string: str = '' + # Get the "target access string" (test mode or otherwise) if _TEST_MODE: _LOGGER.warning('Using FALLBACK_PROPOSAL_ID') - target_access_string = self.__FALLBACK_PROPOSAL_ID + target_access_string: str = self.__FALLBACK_TAS + else: + project: Optional[Project] = Project.objects.filter(id=access_id).first() + if not project: + msg: str = f'Project {access_id} does not exist' + _LOGGER.error(msg) + print(msg) + return Squonk2AgentRv(success=False, msg=msg) + target_access_string = project.title + assert target_access_string - unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(proposal=target_access_string).first() - if not unit: + # Now we check and create a Squonk2Unit... + unit_name_truncated, unit_name_full = self._build_unit_name(target_access_string) + sq2_unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(name=unit_name_full).first() + if not sq2_unit: _LOGGER.info('No existing Unit for this TAS (%s)', target_access_string) - unit_name: styr = 'Fragalysis' \ - f' {self.__CFG_SQUONK2_SLUG}' \ - f' {target_access_string[:_MAX_AUX_STRING_LENGTH]}' - rv: AsApiRv = AsApi.create_unit(unit_name=unit_name, + rv: AsApiRv = AsApi.create_unit(self.__org_owner_as_token, + unit_name=unit_name_truncated, org_id=self.__org_record.uuid, billing_day=self.__unit_billing_day) if not rv.success: @@ -264,21 +395,20 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: return Squonk2AgentRv(success=False, msg=msg) unit_uuid: str = rv.msg['id'] - unit: Squonk2Unit = Squonk2Unit(uuid=unit_uuid, - name=unit_name, - proposal=target_access_string, - organisation=self.__org_record.id) - unit.save() + sq2_unit = Squonk2Unit(uuid=unit_uuid, + name=unit_name_full, + organisation_id=self.__org_record.id) + sq2_unit.save() + + _LOGGER.info('Created NEW Unit %s for TAS (%s)', unit_uuid, target_access_string) else: _LOGGER.debug('Unit %s already exists for this TAS (%s)', - unit.uuid, + sq2_unit.uuid, target_access_string) - return Squonk2AgentRv(success=True, msg=unit.uuid) + return Squonk2AgentRv(success=True, msg=sq2_unit) - def _ensure_project(self, - target_id: int, - user_id: int) -> Squonk2AgentRv: + def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: """Gets or creates a Squonk2 Project, used as the destination of files and job executions. Each project requires an AS Product (tied to the User and Target) and Unit (tied to the proposal). @@ -291,50 +421,64 @@ def _ensure_project(self, For testing the target and user IDs are permitted to be 0. """ + assert params + assert isinstance(params, CommonParams) if not _TEST_MODE: - assert target_id > 0 - assert user_id > 0 + assert params.target_id > 0 + assert params.user_id > 0 # A Squonk2Unit must exist for the Target Access String, and there must be # a Squonk2Project record for the user/target combination. # If not they are created. - rv: Squonk2AgentRv = self._ensure_unit(target_id) + print(f'Checking Unit for access ID {params.access_id}...') + rv: Squonk2AgentRv = self._ensure_unit(params.access_id) if not rv.success: return rv - unit_uuid: str = rv.msg - print(f'Got or Created Unit {unit_uuid}') + unit: Squonk2Unit = rv.msg + print(f'Got or Created Unit {unit.uuid}') # A Squonk2Project record must exist for the unit/user/target combination. # If not it is created. - project: Optional[Squonk2Project] =\ - Squonk2Project.objects.filter(user__id=user_id, - target__id=target_id, - unit__uuid=unit_uuid)\ - .first() - if not project: - # Need to call upon Squonk2 to create a 'Product' and a 'Project'. - # The Product is created by the organisation 'owner' - # the 'Project' is created on behalf of the 'user'. - # - # We create a corresponding Squonk2Project when both have been successful. - rv = self._create_project() + user: Optional[User] = None + user_name: str = self.__DUMMY_USER + session: Optional[SessionProject] = None + session_title: str = self.__DUMMY_SESSION_TITLE + if params.user_id: + user = User.objects.filter(id=params.user_id).first() + user_name = user.username + if params.session_id: + session = SessionProject.objects.filter(id=params.session_id).first() + session_title = session.title + assert user_name + assert session_title + + print(f'Checking Project for User/Session {user_name}/{session_title}...') + name_truncated, name_full = self._build_product_name(user_name, session_title) + sq2_project: Optional[Squonk2Project] = Squonk2Project.objects.filter(name=name_full).first() + if not sq2_project: + _LOGGER.info('No existing Squonk2 Project ("%s")', name_full) + # Need to call upon Squonk2 to create a 'Product' + # (and corresponding 'Product'). + rv = self._create_product_and_project(unit, user_name, session_title, params) if not rv.success: return rv - project = Squonk2Project(uuid=project_uuid, - name=project_name, - product_uuid=product_uuid, - unit=unit_id, - user=user_id, - target=target_id) - project.save() + sq2_project = rv.msg + assert sq2_project + else: + _LOGGER.debug('Project already exists ("%s")', prod_name_truncated) - return Squonk2AgentRv(success=True, msg=project.uuid) + return Squonk2AgentRv(success=True, msg=sq2_project.uuid) def configured(self) -> Squonk2AgentRv: """Returns True if the module appears to be configured, i.e. all the environment variables appear to be set. """ + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + print(msg) + _LOGGER.warning(msg) + # To prevent repeating the checks, all of which are based on # static (environment) variables, if we've been here before # just return our previous result. @@ -347,7 +491,7 @@ def configured(self) -> Squonk2AgentRv: if name.startswith('_Squonk2Agent__CFG_'): if value is None: cfg_name: str = name.split('_Squonk2Agent__CFG_')[1] - msg: str = f'{cfg_name} is not set' + msg = f'{cfg_name} is not set' _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -356,8 +500,8 @@ def configured(self) -> Squonk2AgentRv: # Is the slug too long? # Limited to 10 characters if len(self.__CFG_SQUONK2_SLUG) > _MAX_SLUG_LENGTH: - msg: str = f'Slug is longer than {_MAX_SLUG_LENGTH} characters'\ - f' ({self.__CFG_SQUONK2_SLUG})' + msg = f'Slug is longer than {_MAX_SLUG_LENGTH} characters'\ + f' ({self.__CFG_SQUONK2_SLUG})' _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -408,8 +552,13 @@ def ping(self) -> Squonk2AgentRv: by calling on '_pre_flight_checks()'. If the org is known a Squonk2Org record is created, if not the ping fails. """ + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + print(msg) + _LOGGER.warning(msg) + if not self.configured(): - msg: str = 'Not configured' + msg = 'Not configured' _LOGGER.debug(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -458,38 +607,47 @@ def ping(self) -> Squonk2AgentRv: # Everything's responding if we get here... return SuccessRv - def run_job(self, params: RunJob) -> Squonk2AgentRv: + def run_job(self, params: RunJobParams) -> Squonk2AgentRv: """Executes a Job on a Squonk2 installation. """ assert params - assert isinstance(params, RunJob) + assert isinstance(params, RunJobParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + print(msg) + _LOGGER.warning(msg) # Protect against lack of config or connection/setup issues... if not self.ping(): - msg: str = 'Squonk2 ping failed.'\ - ' Are we configured properly and is Squonk alive?' + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk alive?' return Squonk2AgentRv(success=False, msg=msg) return SuccessRv - def send(self, params: Send) -> Squonk2AgentRv: + def send(self, params: SendParams) -> Squonk2AgentRv: """A blocking method that takes care of sending a set of files to the configured Squonk2 installation. """ assert params - assert isinstance(params, Send) + assert isinstance(params, SendParams) - # Protect against lack of config or connection/setup issues... + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + print(msg) + _LOGGER.warning(msg) + + # Every public API**MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... if not self.ping(): - msg: str = 'Squonk2 ping failed.'\ - ' Are we configured properly and is Squonk alive?' + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk alive?' return Squonk2AgentRv(success=False, msg=msg) - rv_u: Squonk2AgentRv = self._ensure_project(params.user_id, - params.access_id, - params.target_id) - if not rv_u.succes: - msg: str = 'Failed to create corresponding Squonk2 Project' + rv_u: Squonk2AgentRv = self._ensure_project(params.common) + if not rv_u.success: + msg = 'Failed to create corresponding Squonk2 Project' return Squonk2AgentRv(success=False, msg=msg) return SuccessRv From 3bb94be95aa965f49e0a365b0017b1b79fc40ac8 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 8 Dec 2022 14:21:13 +0000 Subject: [PATCH 010/112] Initial unit/product/project creation --- docker-compose.yml | 2 ++ viewer/squonk2_agent.py | 64 ++++++++++++++++------------------------- 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ae4d0eca..1da22b12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,8 @@ services: POSTGRESQL_PASSWORD: fragalysis POSTGRESQL_HOST: database POSTGRESQL_PORT: 5432 + # Logging level + LOGGING_FRAMEWORK_ROOT_LEVEL: ${LOGGING_FRAMEWORK_ROOT_LEVEL:-INFO} # Celery tasks run synchronously? CELERY_TASK_ALWAYS_EAGER: 'True' # Default Debug to true for local development diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 25e180e0..f5e2db76 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -287,7 +287,9 @@ def _create_product_and_project(self, user_name: str, session_title: str, params: CommonParams) -> Squonk2AgentRv: - """Called if a Product (and Project) needs to be created. + """Called if a Product (and Project) needs to be created. If successful, + this call returns a dictionary in the response "msg" that contains values + for the keys "sq2_product_uuid" and "sq2_project_uuid". """ assert unit assert user_name @@ -296,8 +298,7 @@ def _create_product_and_project(self, # Create an AS Product. name_truncated, name_full = self._build_product_name(user_name, session_title) - msg: str = f'Creating AS Product "{name_full}"...' - print(msg) + msg: str = f'Creating AS Product "{name_truncated}"...' _LOGGER.info(msg) as_rv: AsApiRv = AsApi.create_product(self.__org_owner_as_token, @@ -305,53 +306,39 @@ def _create_product_and_project(self, unit_id=unit.uuid, product_type=_SQ2_PRODUCT_TYPE, flavour=_SQ2_PRODUCT_FLAVOUR) - print(as_rv) assert as_rv.success product_uuid: str = as_rv.msg['id'] msg = f'Created AS Product "{product_uuid}"...' - print(msg) _LOGGER.info(msg) # Create a DM Project name_truncated, name_full = self._build_project_name(1, 2) - msg = f'Creating DM Project "{name_full}"...' - print(msg) + msg = f'Creating DM Project "{name_truncated}"...' _LOGGER.info(msg) dm_rv: DmApiRv = DmApi.create_project(self.__org_owner_dm_token, project_name=name_truncated, as_tier_product_id=product_uuid) - print(dm_rv) assert dm_rv.success project_uuid: str = dm_rv.msg["project_id"] msg = f'Created DM Project "{project_uuid}"...' - print(msg) _LOGGER.info(msg) # Add the user as an Editor to the Project msg = f'Adding "{user_name} to DM Project as Editor...' - print(msg) _LOGGER.info(msg) dm_rv = DmApi.add_project_editor(self.__org_owner_dm_token, project_id=project_uuid, editor=user_name) - print(dm_rv) assert dm_rv.success msg = f'Added "{user_name} to DM Project as Editor' - print(msg) _LOGGER.info(msg) - # Now record these remote objects in a new - # Squonk2Project record... - sq2_project = Squonk2Project(uuid=project_uuid, - name=name_full, - product_uuid=product_uuid, - unit_id=unit.id) - sq2_project.save() - # If the second call fails - delete the object created in the first - return Squonk2AgentRv(success=True, msg=sq2_project) + response_msg: Dict[str, Any] = {"sq2_project_uuid": project_uuid, + "sq2_product_uuid": product_uuid} + return Squonk2AgentRv(success=True, msg=response_msg) def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: """Gets or creates a Squonk2 Unit based on a customer's "target access string" @@ -375,7 +362,6 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: if not project: msg: str = f'Project {access_id} does not exist' _LOGGER.error(msg) - print(msg) return Squonk2AgentRv(success=False, msg=msg) target_access_string = project.title assert target_access_string @@ -417,7 +403,7 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: check whether the user/proposal combination - it assumes that what's been given has been checked. - On success the returned message is used to carry the Squonk2 project UUID. + On success the returned message is used to carry the Squonk2Project record. For testing the target and user IDs are permitted to be 0. """ @@ -427,16 +413,11 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: assert params.target_id > 0 assert params.user_id > 0 - # A Squonk2Unit must exist for the Target Access String, and there must be - # a Squonk2Project record for the user/target combination. - # If not they are created. - - print(f'Checking Unit for access ID {params.access_id}...') + # A Squonk2Unit must exist for the Target Access String. rv: Squonk2AgentRv = self._ensure_unit(params.access_id) if not rv.success: return rv unit: Squonk2Unit = rv.msg - print(f'Got or Created Unit {unit.uuid}') # A Squonk2Project record must exist for the unit/user/target combination. # If not it is created. @@ -453,22 +434,31 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: assert user_name assert session_title - print(f'Checking Project for User/Session {user_name}/{session_title}...') name_truncated, name_full = self._build_product_name(user_name, session_title) sq2_project: Optional[Squonk2Project] = Squonk2Project.objects.filter(name=name_full).first() if not sq2_project: - _LOGGER.info('No existing Squonk2 Project ("%s")', name_full) + msg = f'No existing Squonk2Project for "{name_full}"' + _LOGGER.info(msg) # Need to call upon Squonk2 to create a 'Product' # (and corresponding 'Product'). rv = self._create_product_and_project(unit, user_name, session_title, params) if not rv.success: return rv - sq2_project = rv.msg - assert sq2_project + # Now record these new remote objects in a new + # Squonk2Project record. As it's worked we're given + # a dictionary with keys "sq2_project_uuid" and "sq2_product_uuid" + sq2_project = Squonk2Project(uuid=rv.msg['sq2_project_uuid'], + name=name_full, + product_uuid=rv.msg['sq2_product_uuid'], + unit_id=unit.id) + sq2_project.save() + msg = f'Created NEW Squonk2Project for "{name_full}"' + _LOGGER.info(msg) else: - _LOGGER.debug('Project already exists ("%s")', prod_name_truncated) + msg = f'Squonk2Project already exists ("{name_full}")' + _LOGGER.debug(msg) - return Squonk2AgentRv(success=True, msg=sq2_project.uuid) + return Squonk2AgentRv(success=True, msg=sq2_project) def configured(self) -> Squonk2AgentRv: """Returns True if the module appears to be configured, @@ -476,7 +466,6 @@ def configured(self) -> Squonk2AgentRv: """ if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' - print(msg) _LOGGER.warning(msg) # To prevent repeating the checks, all of which are based on @@ -554,7 +543,6 @@ def ping(self) -> Squonk2AgentRv: """ if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' - print(msg) _LOGGER.warning(msg) if not self.configured(): @@ -615,7 +603,6 @@ def run_job(self, params: RunJobParams) -> Squonk2AgentRv: if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' - print(msg) _LOGGER.warning(msg) # Protect against lack of config or connection/setup issues... @@ -635,7 +622,6 @@ def send(self, params: SendParams) -> Squonk2AgentRv: if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' - print(msg) _LOGGER.warning(msg) # Every public API**MUST** call ping(). From 7007f830ea40b830b2a71fb80c3dbf70adccf662 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 8 Dec 2022 16:02:43 +0000 Subject: [PATCH 011/112] Removal of Project/Product on error More consistent logging --- requirements.txt | 1 + viewer/squonk2_agent.py | 123 +++++++++++++++++++++++++++++++--------- 2 files changed, 97 insertions(+), 27 deletions(-) diff --git a/requirements.txt b/requirements.txt index 58b164bf..613effa4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -121,6 +121,7 @@ validators==0.17.1 vine==1.3.0 wcwidth==0.2.5 websocket-client==0.57.0 +wrapt==1.14.1 xchem-db==0.1.26b0 yarl==1.5.1 zipp==3.1.0 diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index f5e2db76..ab96aa56 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -194,7 +194,8 @@ def _get_squonk2_owner_tokens(self) -> Optional[Tuple[str, str]]: """ assert self.__keycloak_hostname - _LOGGER.info('__keycloak_hostname="%s" __keycloak_realm="%s" dm-client=%s as-client=%s org_owner=%s', + _LOGGER.debug('__keycloak_hostname="%s" __keycloak_realm="%s"' + ' dm-client=%s as-client=%s org_owner=%s', self.__keycloak_hostname, self.__keycloak_realm, self.__CFG_OIDC_DM_CLIENT_ID, @@ -241,22 +242,24 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: if squonk2_org and squonk2_org.uuid != self.__CFG_SQUONK2_ORG_UUID: msg: str = f'Configured Squonk2 Organisation ({self.__CFG_SQUONK2_ORG_UUID})'\ f' does not match pre-existing record ({squonk2_org.uuid})' + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) # OK, so the ORG UUID has not changed. # Is it known to the configured AS? if not self._get_squonk2_owner_tokens(): msg = 'Failed to get AS or DM token for organisation owner' + _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) # Get the ORG from the AS API. # If it knows the org the response will be successful, # and we'll also have the Org's name. - _LOGGER.info('Checking organisation (%s)', self.__CFG_SQUONK2_ORG_UUID) as_o_rv = AsApi.get_organisation(self.__org_owner_as_token, org_id=self.__CFG_SQUONK2_ORG_UUID) if not as_o_rv.success: - msg = 'Failed checking AS Organisation' + msg = 'Failed to get AS Organisation' + _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) # The org is known to the AS. @@ -264,17 +267,28 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: as_v_rv: AsApiRv = AsApi.get_version() if not as_v_rv.success: msg = 'Failed to get version from AS' + _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) + as_version: str = as_v_rv.msg['version'] # If there's no Squonk2Org record, create one, # recording the ORG ID and the AS we used to verify it exists. if not squonk2_org: + _LOGGER.info('Creating NEW Squonk2Org record for %s.' + ' as-url=%s as-name="%s" as-version=%s', + self.__CFG_SQUONK2_ORG_UUID, + self.__CFG_SQUONK2_ASAPI_URL, + as_o_rv.msg['name'], + as_version) squonk2_org = Squonk2Org(uuid=self.__CFG_SQUONK2_ORG_UUID, name=as_o_rv.msg['name'], as_url=self.__CFG_SQUONK2_ASAPI_URL, as_version=as_version) squonk2_org.save() + else: + _LOGGER.debug('Squonk2Org record already exists for %s', + self.__CFG_SQUONK2_ORG_UUID) # Keep the record ID for future use. self.__org_record = squonk2_org @@ -282,6 +296,34 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # Organisation is known to AS, and it hasn't changed. return SuccessRv + def _delete_as_product(self, product_uuid: str) -> None: + """Used in error conditions to remove a previously created Product. + If this fails there's nothing else we can do so we just return regardless. + """ + _LOGGER.warning('Deleting AS Product %s', product_uuid) + + as_rv: AsApiRv = AsApi.delete_product(self.__org_owner_as_token, + product_id=product_uuid) + if not as_rv.success: + _LOGGER.error('Failed to delete AS Product %s', product_uuid) + return + + _LOGGER.warning('Deleted AS Product %s', product_uuid) + + def _delete_dm_project(self, project_uuid: str) -> None: + """Used in error conditions to remove a previously created Project. + If this fails there's nothing else we can do so we just return regardless. + """ + _LOGGER.warning('Deleting DM Project %s', project_uuid) + + dm_rv: DmApiRv = DmApi.delete_project(self.__org_owner_dm_token, + project_id=project_uuid) + if not as_rv.success: + _LOGGER.error('Failed to delete DM Project %s', project_uuid) + return + + _LOGGER.warning('Deleted DM Project %s', project_uuid) + def _create_product_and_project(self, unit: Squonk2Unit, user_name: str, @@ -306,7 +348,11 @@ def _create_product_and_project(self, unit_id=unit.uuid, product_type=_SQ2_PRODUCT_TYPE, flavour=_SQ2_PRODUCT_FLAVOUR) - assert as_rv.success + if not as_rv.success: + msg = f'Failed to create AS Product ({as_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + product_uuid: str = as_rv.msg['id'] msg = f'Created AS Product "{product_uuid}"...' _LOGGER.info(msg) @@ -319,19 +365,34 @@ def _create_product_and_project(self, dm_rv: DmApiRv = DmApi.create_project(self.__org_owner_dm_token, project_name=name_truncated, as_tier_product_id=product_uuid) - assert dm_rv.success + if not dm_rv.success: + msg = f'Failed to create DM Project ({dm_rv.msg})' + _LOGGER.error(msg) + # First delete the AS Product it should have been attached to + self._delete_as_product(product_uuid) + # Then leave... + return Squonk2AgentRv(success=False, msg=msg) + project_uuid: str = dm_rv.msg["project_id"] msg = f'Created DM Project "{project_uuid}"...' _LOGGER.info(msg) # Add the user as an Editor to the Project - msg = f'Adding "{user_name} to DM Project as Editor...' + msg = f'Adding "{user_name}" to DM Project {project_uuid} as Editor...' _LOGGER.info(msg) dm_rv = DmApi.add_project_editor(self.__org_owner_dm_token, project_id=project_uuid, editor=user_name) - assert dm_rv.success - msg = f'Added "{user_name} to DM Project as Editor' + if not dm_rv.success: + msg = f'Failed to add "{user_name}" to DM Project ({dm_rv.msg})' + _LOGGER.error(msg) + # First delete the DM Project amd the corresponding AS Product... + self._delete_dm_project(project_uuid) + self._delete_as_product(product_uuid) + # Then leave... + return Squonk2AgentRv(success=False, msg=msg) + + msg = f'Added "{user_name} to DM Project {project_uuid} as Editor' _LOGGER.info(msg) # If the second call fails - delete the object created in the first @@ -370,14 +431,14 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: unit_name_truncated, unit_name_full = self._build_unit_name(target_access_string) sq2_unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(name=unit_name_full).first() if not sq2_unit: - _LOGGER.info('No existing Unit for this TAS (%s)', target_access_string) + _LOGGER.info('No existing Unit for "%s"', target_access_string) rv: AsApiRv = AsApi.create_unit(self.__org_owner_as_token, unit_name=unit_name_truncated, org_id=self.__org_record.uuid, billing_day=self.__unit_billing_day) if not rv.success: msg: str = rv.msg['error'] - _LOGGER.error('Failed to create Unit for TAS (%s)', target_access_string) + _LOGGER.error('Failed to create Unit "%s"', target_access_string) return Squonk2AgentRv(success=False, msg=msg) unit_uuid: str = rv.msg['id'] @@ -386,9 +447,9 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: organisation_id=self.__org_record.id) sq2_unit.save() - _LOGGER.info('Created NEW Unit %s for TAS (%s)', unit_uuid, target_access_string) + _LOGGER.info('Created NEW Unit %s for "%s"', unit_uuid, target_access_string) else: - _LOGGER.debug('Unit %s already exists for this TAS (%s)', + _LOGGER.debug('Unit %s already exists for "%s" - nothing to do', sq2_unit.uuid, target_access_string) @@ -443,7 +504,10 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: # (and corresponding 'Product'). rv = self._create_product_and_project(unit, user_name, session_title, params) if not rv.success: + msg = f'Failed creating AS Product or DM Project ({rv.msg})' + _LOGGER.error(msg) return rv + # Now record these new remote objects in a new # Squonk2Project record. As it's worked we're given # a dictionary with keys "sq2_project_uuid" and "sq2_product_uuid" @@ -460,13 +524,11 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: return Squonk2AgentRv(success=True, msg=sq2_project) + @synchronized def configured(self) -> Squonk2AgentRv: """Returns True if the module appears to be configured, i.e. all the environment variables appear to be set. """ - if _TEST_MODE: - msg: str = 'Squonk2Agent is in TEST mode' - _LOGGER.warning(msg) # To prevent repeating the checks, all of which are based on # static (environment) variables, if we've been here before @@ -481,7 +543,7 @@ def configured(self) -> Squonk2AgentRv: if value is None: cfg_name: str = name.split('_Squonk2Agent__CFG_')[1] msg = f'{cfg_name} is not set' - _LOGGER.warning(msg) + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) # If we get here all the required configuration variables are set @@ -491,7 +553,7 @@ def configured(self) -> Squonk2AgentRv: if len(self.__CFG_SQUONK2_SLUG) > _MAX_SLUG_LENGTH: msg = f'Slug is longer than {_MAX_SLUG_LENGTH} characters'\ f' ({self.__CFG_SQUONK2_SLUG})' - _LOGGER.warning(msg) + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) # Extract hostname and realm from the legacy variable @@ -533,6 +595,7 @@ def configured(self) -> Squonk2AgentRv: return SuccessRv + @synchronized def ping(self) -> Squonk2AgentRv: """Returns True if all the Squonk2 installations referred to by the URLs respond. @@ -547,7 +610,7 @@ def ping(self) -> Squonk2AgentRv: if not self.configured(): msg = 'Not configured' - _LOGGER.debug(msg) + _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) # Check the UI, DM and AS... @@ -557,10 +620,10 @@ def ping(self) -> Squonk2AgentRv: try: resp: Response = requests.head(url, verify=self.__verify_certificates) except: - _LOGGER.warning('Exception checking UI at %s', url) + _LOGGER.error('Exception checking UI at %s', url) if resp is None or resp.status_code != 200: - msg = f'UI is not responding from {url}' - _LOGGER.debug(msg) + msg = f'Squonk2 UI is not responding from {url}' + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) resp = None @@ -568,10 +631,10 @@ def ping(self) -> Squonk2AgentRv: try: resp = requests.head(url, verify=self.__verify_certificates) except Exception as ex: - _LOGGER.warning('Exception checking DM at %s', url) + _LOGGER.error('Exception checking DM at %s', url) if resp is None or resp.status_code != 308: - msg = f'Data Manager is not responding from {url}' - _LOGGER.debug(msg) + msg = f'Squonk2 DM is not responding from {url}' + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) resp = None @@ -579,10 +642,10 @@ def ping(self) -> Squonk2AgentRv: try: resp = requests.head(url, verify=self.__verify_certificates) except: - _LOGGER.warning('Exception checking AS at %s', url) + _LOGGER.error('Exception checking AS at %s', url) if resp is None or resp.status_code != 308: - msg = f'Account Manager is not responding from {url}' - _LOGGER.debug(msg) + msg = f'Squonk2 AS is not responding from {url}' + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) # OK so far. @@ -590,11 +653,13 @@ def ping(self) -> Squonk2AgentRv: status, msg = self._pre_flight_checks() if not status: msg = f'Failed pre-flight checks ({msg})' + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) # Everything's responding if we get here... return SuccessRv + @synchronized def run_job(self, params: RunJobParams) -> Squonk2AgentRv: """Executes a Job on a Squonk2 installation. """ @@ -609,10 +674,12 @@ def run_job(self, params: RunJobParams) -> Squonk2AgentRv: if not self.ping(): msg = 'Squonk2 ping failed.'\ ' Are we configured properly and is Squonk alive?' + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) return SuccessRv + @synchronized def send(self, params: SendParams) -> Squonk2AgentRv: """A blocking method that takes care of sending a set of files to the configured Squonk2 installation. @@ -629,11 +696,13 @@ def send(self, params: SendParams) -> Squonk2AgentRv: if not self.ping(): msg = 'Squonk2 ping failed.'\ ' Are we configured properly and is Squonk alive?' + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) rv_u: Squonk2AgentRv = self._ensure_project(params.common) if not rv_u.success: msg = 'Failed to create corresponding Squonk2 Project' + _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) return SuccessRv From 9de3e52a83a93bc4bd0926d81c6ebd84f3fceb9f Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 8 Dec 2022 16:22:53 +0000 Subject: [PATCH 012/112] Log tweak --- viewer/squonk2_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index ab96aa56..f458a652 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -287,7 +287,7 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: as_version=as_version) squonk2_org.save() else: - _LOGGER.debug('Squonk2Org record already exists for %s', + _LOGGER.debug('Squonk2Org already exists for %s - nothing to do', self.__CFG_SQUONK2_ORG_UUID) # Keep the record ID for future use. @@ -519,7 +519,7 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: msg = f'Created NEW Squonk2Project for "{name_full}"' _LOGGER.info(msg) else: - msg = f'Squonk2Project already exists ("{name_full}")' + msg = f'Squonk2Project already exists "{name_full}" - nothing to do' _LOGGER.debug(msg) return Squonk2AgentRv(success=True, msg=sq2_project) From 6885e1c9e4e74d179f41690028621a721227a31c Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 12 Dec 2022 11:13:37 +0000 Subject: [PATCH 013/112] Adds new ensure_project() method - for use in existing code for slower migration --- viewer/squonk2_agent.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index f458a652..8a666d43 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -276,7 +276,7 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # recording the ORG ID and the AS we used to verify it exists. if not squonk2_org: _LOGGER.info('Creating NEW Squonk2Org record for %s.' - ' as-url=%s as-name="%s" as-version=%s', + ' as-url=%s as-org="%s" as-version=%s', self.__CFG_SQUONK2_ORG_UUID, self.__CFG_SQUONK2_ASAPI_URL, as_o_rv.msg['name'], @@ -673,7 +673,7 @@ def run_job(self, params: RunJobParams) -> Squonk2AgentRv: # Protect against lack of config or connection/setup issues... if not self.ping(): msg = 'Squonk2 ping failed.'\ - ' Are we configured properly and is Squonk alive?' + ' Are we configured properly and is Squonk2 alive?' _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -695,7 +695,7 @@ def send(self, params: SendParams) -> Squonk2AgentRv: # This ensures Squonk2 is available and gets suitable API tokens... if not self.ping(): msg = 'Squonk2 ping failed.'\ - ' Are we configured properly and is Squonk alive?' + ' Are we configured properly and is Squonk2 alive?' _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -707,6 +707,37 @@ def send(self, params: SendParams) -> Squonk2AgentRv: return SuccessRv + @synchronized + def ensure_project(self, params: CommonParams) -> Squonk2AgentRv: + """A blocking method that takes care of the provisioning of the + required Squonk2 environment. For Fragalysis this entails the + creation of a 'Squonk2 Project' (which also requires a 'Unit' and 'Product'). + + If successful the Corresponding Squonk2Project record is returned as + the response 'msg' value. + """ + assert params + assert isinstance(params, CommonParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API**MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + rv_u: Squonk2AgentRv = self._ensure_project(params) + if not rv_u.success: + msg = 'Failed to create corresponding Squonk2 Project' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return rv_u # The global (singleton). # This acts as out sole singleton, From ab4561853bb5b8965c01b134d8cc66a6a13b95ab Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 12 Dec 2022 11:27:19 +0000 Subject: [PATCH 014/112] Fix product-tier flavour --- docker-compose.yml | 2 +- viewer/squonk2_agent.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1da22b12..f5f01ced 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: OIDC_RP_CLIENT_SECRET: 'c6245428-04c7-466f-9c4f-58c340e981c2' SQUONK2_VERIFY_CERTIFICATES: 'No' SQUONK2_UNIT_BILLING_DAY: 3 - SQUONK2_PRODUCT_FLAVOUR: BRONZE + SQUONK2_PRODUCT_FLAVOUR: GOLD SQUONK2_SLUG: fs-local # The following are normally populated via a local '.env' file # that is not part of the repo, it is a file you create diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 8a666d43..11e88821 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -55,12 +55,17 @@ _SUPPORTED_PRODUCT_FLAVOURS: List[str] = ["BRONZE", "SILVER", "GOLD"] +# Squonk2 Have defined limits - assumed here. +# verify with your Squonk2 installation. _MAX_SQ2_NAME_LENGTH: int = 80 + +# A slug used for names this Fragalysis will create +# and a prefix string. So Squonk2 objects will be called "Fragalysis {slug}" _MAX_SLUG_LENGTH: int = 10 _SQ2_NAME_PREFIX: str = "Fragalysis" +# Built-in _SQ2_PRODUCT_TYPE: str = 'DATA_MANAGER_PROJECT_TIER_SUBSCRIPTION' -_SQ2_PRODUCT_FLAVOUR: str = 'GOLD' # How long are Squonk2 'names'? _SQ2_MAX_NAME_LENGTH: int = 80 @@ -130,8 +135,6 @@ def __init__(self): # The integer billing day, valid if greater than zero self.__unit_billing_day: int = 0 - # The product tier, valid if set - self.__product_flavour: str = '' # True if configured... self.__configuration_checked: bool = False self.__configured: bool = False @@ -347,7 +350,7 @@ def _create_product_and_project(self, product_name=name_truncated, unit_id=unit.uuid, product_type=_SQ2_PRODUCT_TYPE, - flavour=_SQ2_PRODUCT_FLAVOUR) + flavour=self.__CFG_SQUONK2_PRODUCT_FLAVOUR) if not as_rv.success: msg = f'Failed to create AS Product ({as_rv.msg})' _LOGGER.error(msg) @@ -519,7 +522,7 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: msg = f'Created NEW Squonk2Project for "{name_full}"' _LOGGER.info(msg) else: - msg = f'Squonk2Project already exists "{name_full}" - nothing to do' + msg = f'Project {sq2_project.uuid} already exists for "{name_full}" - nothing to do' _LOGGER.debug(msg) return Squonk2AgentRv(success=True, msg=sq2_project) @@ -576,13 +579,13 @@ def configured(self) -> Squonk2AgentRv: _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - # Product tier to upper-case + # Product tier flavour must be one of a known value. + # It's stored in the object's self.__product_flavour as an upper-case value if not self.__CFG_SQUONK2_PRODUCT_FLAVOUR in _SUPPORTED_PRODUCT_FLAVOURS: msg = f'SQUONK2_PRODUCT_FLAVOUR ({self.__CFG_SQUONK2_PRODUCT_FLAVOUR})' \ ' is not supported' _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - self.__product_flavour = self.__CFG_SQUONK2_PRODUCT_FLAVOUR.upper() # Don't verify Squonk2 SSL certificates? if self.__SQUONK2_VERIFY_CERTIFICATES and self.__SQUONK2_VERIFY_CERTIFICATES.lower() == 'no': From 411dc3a51153956de7c06800e1e01fe00082d893 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 12 Dec 2022 12:09:29 +0000 Subject: [PATCH 015/112] Better TEST mode --- docker-compose.yml | 4 +- viewer/squonk2_agent.py | 88 +++++++++++++++++++++++++---------------- 2 files changed, 57 insertions(+), 35 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f5f01ced..e00dbd7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: OIDC_RP_CLIENT_SECRET: 'c6245428-04c7-466f-9c4f-58c340e981c2' SQUONK2_VERIFY_CERTIFICATES: 'No' SQUONK2_UNIT_BILLING_DAY: 3 - SQUONK2_PRODUCT_FLAVOUR: GOLD + SQUONK2_PRODUCT_FLAVOUR: BRONZE SQUONK2_SLUG: fs-local # The following are normally populated via a local '.env' file # that is not part of the repo, it is a file you create @@ -84,7 +84,7 @@ services: SQUONK2_UI_URL: ${SQUONK2_UI_URL} SQUONK2_DMAPI_URL: ${SQUONK2_DMAPI_URL} SQUONK2_ASAPI_URL: ${SQUONK2_ASAPI_URL} - FALLBACK_TAS: ${FALLBACK_TAS} + DUMMY_TAS: ${DUMMY_TAS} DUMMY_USER: ${DUMMY_USER} DUMMY_SESSION_TITLE: ${DUMMY_SESSION_TITLE} ports: diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 11e88821..ec028cc6 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -126,10 +126,8 @@ def __init__(self): os.environ.get('DUMMY_SESSION_TITLE') self.__DUMMY_USER: Optional[str] =\ os.environ.get('DUMMY_USER') - self.__FALLBACK_TAS: Optional[str] =\ - os.environ.get('FALLBACK_TAS') - self.__FORCE_FALLBACK_TAS: Optional[str] =\ - os.environ.get('FORCE_FALLBACK_TAS') + self.__DUMMY_TAS: Optional[str] =\ + os.environ.get('DUMMY_TAS') self.__SQUONK2_VERIFY_CERTIFICATES: Optional[str] = \ os.environ.get('SQUONK2_VERIFY_CERTIFICATES') @@ -158,6 +156,55 @@ def __init__(self): self.__keycloak_hostname: str = '' self.__keycloak_realm: str = '' + def _get_user_name(self, user_id: int) -> str: + # Gets the username (if id looks sensible) + # or a fixed value (used for testing) + if user_id == 0: + assert _TEST_MODE + _LOGGER.warning('Caution - in TEST mode, using __DUMMY_USER (%s)', + self.__DUMMY_USER) + + user_name: str = self.__DUMMY_USER + if user_id: + user: User = User.objects.filter(id=params.user_id).first() + assert user + user_name = user.username + assert user_name + return user_name + + def _get_target_access_string(self, access_id: int) -> str: + # Gets the Target Access String (if id looks sensible) + # or a fixed value (used for testing) + if access_id == 0: + assert _TEST_MODE + _LOGGER.warning('Caution - in TEST mode, using __DUMMY_TAS (%s)', + self.__DUMMY_TAS) + + # Get the "target access string" (test mode or otherwise) + target_access_string: str = self.__DUMMY_TAS + if access_id: + project: Optional[Project] = Project.objects.filter(id=access_id).first() + assert project + target_access_string = project.title + assert target_access_string + return target_access_string + + def _get_session_title(self, session_id: int) -> str: + # Gets the Session title (if id looks sensible) + # or a fixed value (used for testing) + if session_id == 0: + assert _TEST_MODE + _LOGGER.warning('Caution - in TEST mode, using __DUMMY_TAS__DUMMY_SESSION_TITLE (%s)', + self.__DUMMY_SESSION_TITLE) + + session_title: str = self.__DUMMY_SESSION_TITLE + if session_id: + session_project: SessionProject = SessionProject.objects.filter(id=params.session_id).first() + assert session_project + session_title = session_project.title + assert session_title + return session_title + def _build_unit_name(self, target_access_string: str) -> Tuple[str, str]: assert target_access_string # AS Units are named using the Target Access String (TAS) @@ -414,20 +461,8 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: On success the returned message is used to carry the Squonk2 project UUID. """ assert self.__org_record - if not _TEST_MODE: - assert access_id - # Get the "target access string" (test mode or otherwise) - if _TEST_MODE: - _LOGGER.warning('Using FALLBACK_PROPOSAL_ID') - target_access_string: str = self.__FALLBACK_TAS - else: - project: Optional[Project] = Project.objects.filter(id=access_id).first() - if not project: - msg: str = f'Project {access_id} does not exist' - _LOGGER.error(msg) - return Squonk2AgentRv(success=False, msg=msg) - target_access_string = project.title + target_access_string = self._get_target_access_string(access_id) assert target_access_string # Now we check and create a Squonk2Unit... @@ -461,7 +496,7 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: """Gets or creates a Squonk2 Project, used as the destination of files and job executions. Each project requires an AS Product - (tied to the User and Target) and Unit (tied to the proposal). + (tied to the User and Session) and Unit (tied to the Proposal/Project). The proposal is expected to be valid for a given user, this method does not check whether the user/proposal combination - it assumes that what's been @@ -473,9 +508,6 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: """ assert params assert isinstance(params, CommonParams) - if not _TEST_MODE: - assert params.target_id > 0 - assert params.user_id > 0 # A Squonk2Unit must exist for the Target Access String. rv: Squonk2AgentRv = self._ensure_unit(params.access_id) @@ -483,18 +515,8 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: return rv unit: Squonk2Unit = rv.msg - # A Squonk2Project record must exist for the unit/user/target combination. - # If not it is created. - user: Optional[User] = None - user_name: str = self.__DUMMY_USER - session: Optional[SessionProject] = None - session_title: str = self.__DUMMY_SESSION_TITLE - if params.user_id: - user = User.objects.filter(id=params.user_id).first() - user_name = user.username - if params.session_id: - session = SessionProject.objects.filter(id=params.session_id).first() - session_title = session.title + user_name: str = self._get_user_name(params.user_id) + session_title: str = self._get_session_title(params.session_id) assert user_name assert session_title From 8ba4c83cc8875610e7d6fbc04d54ddb240525df7 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 12 Dec 2022 13:51:44 +0000 Subject: [PATCH 016/112] Now able to get UI URL form agent Removed old settings variables (now using SQ agent) --- fragalysis/settings.py | 6 --- viewer/squonk2_agent.py | 9 +++- viewer/squonk_job_file_upload.py | 5 ++- viewer/squonk_job_request.py | 48 +++++++++++++++------ viewer/views.py | 72 ++++++++++++++++++++++---------- 5 files changed, 98 insertions(+), 42 deletions(-) diff --git a/fragalysis/settings.py b/fragalysis/settings.py index 5e6bfeed..bf755303 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -339,12 +339,6 @@ # dedicated Discourse server. DISCOURSE_DEV_POST_SUFFIX = os.environ.get("DISCOURSE_DEV_POST_SUFFIX", '') -# Squonk settings for API calls to Squonk Platform. -# The environment variable SQUONK2_DMAPI_URL -# is expected by the squonk2-client package. -SQUONK2_DMAPI_URL = os.environ.get('SQUONK2_DMAPI_URL') -SQUONK2_UI_URL = os.environ.get('SQUONK2_UI_URL') - SQUONK2_MEDIA_DIRECTORY = "fragalysis-files" SQUONK2_INSTANCE_API = "data-manager-ui/results/instance/" diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index ec028cc6..c1b0370e 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -32,8 +32,7 @@ # Parameters common to each named Tuple CommonParams: namedtuple = namedtuple("CommonParams", - ["token", - "user_id", + ["user_id", "access_id", "target_id", "session_id"]) @@ -549,6 +548,12 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: return Squonk2AgentRv(success=True, msg=sq2_project) + @synchronized + def get_ui_url(self): + """Returns the UI URL, if configured. + """ + return self.__CFG_SQUONK2_UI_URL + @synchronized def configured(self) -> Squonk2AgentRv: """Returns True if the module appears to be configured, diff --git a/viewer/squonk_job_file_upload.py b/viewer/squonk_job_file_upload.py index 37ac7d34..a8abdcf0 100644 --- a/viewer/squonk_job_file_upload.py +++ b/viewer/squonk_job_file_upload.py @@ -23,9 +23,12 @@ add_prop_to_sdf, create_media_sub_directory ) +from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent logger = get_task_logger(__name__) +_SQ2A: Squonk2Agent = get_squonk2_agent() + # A "Blank" molecule. # Inserted at the top of SDF files pulled from Squonk2. # This provides a 'header' that is then used @@ -79,7 +82,7 @@ def _insert_sdf_blank_mol(job_request, transition_time, sdf_filename): # Compound set reference URL. # What's the https-prefixed URL to the instance? # The record's URL is relative to the API. - ref_url = settings.SQUONK2_UI_URL + ref_url = _SQ2A.get_ui_url() if ref_url.endswith('/'): ref_url += job_request.squonk_url_ext else: diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index f0484f97..5ef260ab 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -16,23 +16,21 @@ JobRequest, JobFileTransfer ) from viewer.utils import get_https_host +from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent logger = logging.getLogger(__name__) +_SQ2A: Squonk2Agent = get_squonk2_agent() def check_squonk_active(request): - """Call a Squonk API to check that Squonk can be reached. + """Call the Squonk2 Agent to check that Squonk2 can be reached. """ - logger.info('+ Ping') - auth_token = request.session['oidc_access_token'] - - logger.info('oidc token') - logger.info(auth_token) + logger.info('+ Squonk2Agent.ping()') - result = DmApi.ping(auth_token) - logger.debug(result) + ping_rv = _SQ2A.ping() + logger.debug(ping_rv) - return result.success + return ping_rv.success def get_squonk_job_config(request, @@ -94,13 +92,16 @@ def create_squonk_job(request): squonk_job_name = request.data['squonk_job_name'] target_id = request.data['target'] snapshot_id = request.data['snapshot'] - squonk_project = request.data['squonk_project'] + session_project_id = request.data['session_project'] + # The access ID (legacy Project record ID) + access_id = request.data['access'] squonk_job_spec = request.data['squonk_job_spec'] logger.info('+ squonk_job_name=%s', squonk_job_name) logger.info('+ target_id=%s', target_id) logger.info('+ snapshot_id=%s', snapshot_id) - logger.info('+ squonk_project=%s', squonk_project) + logger.info('+ session_project_id=%s', session_project_id) + logger.info('+ access_id=%s', access_id) logger.info('+ squonk_job_spec=%s', squonk_job_spec) job_transfers = JobFileTransfer.objects.filter(snapshot=snapshot_id) @@ -116,12 +117,35 @@ def create_squonk_job(request): job_transfer.transfer_status) raise ValueError('Job Transfer not complete') + # This requires a Squonk2 Project (created by the Squonk2Agent). + # It may be an existing project, or it might be a new project. + common_params = CommonParams(user_id=user.id, + access_id=access_id, + target_id=target_id, + session_id=session_project_id) + sq2_rv = _SQ2A.ensure_project(common_params) + if not sq2_rv.success: + msg = f'JobTransfer failed to get a Squonk2 Project' \ + f' for User {user.username}, Access ID {access_id},' \ + f' Target ID {target_id}, and SessionProject ID {session_id}.' \ + f' Returning the message "{ep_rv.msg}".' \ + ' Cannot continue' + content = {'message': msg} + logger.error(msg) + return Response(content, + status=status.HTTP_404_NOT_FOUND) + # The project UUID is in the response msg (a Squonk2Project object) + squonk2_project = sq2_rv.msg.uuid + job_request = JobRequest() job_request.squonk_job_name = squonk_job_name job_request.user = request.user job_request.snapshot = Snapshot.objects.get(id=snapshot_id) job_request.target = Target.objects.get(id=target_id) - job_request.squonk_project = squonk_project + # We should use a foreign key, + # but to avoid migration issues with the existing code + # we continue to use the project UUID string field. + job_request.squonk_project = squonk2_project job_request.squonk_job_spec = squonk_job_spec # Saving creates the uuid for the callback diff --git a/viewer/views.py b/viewer/views.py index 0d392787..ff56c187 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -62,6 +62,9 @@ JobFileTransfer ) from viewer import filters +from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent +from viewer.squonk2_agent import CommonParams + from .forms import CSetForm, CSetUpdateForm, TSetForm from .tasks import ( check_services, @@ -141,6 +144,7 @@ _SESSION_ERROR = 'session_error' _SESSION_MESSAGE = 'session_message' +_SQ2A: Squonk2Agent = get_squonk2_agent() class VectorsView(ISpyBSafeQuerySet): """ DjagnoRF view for vectors @@ -641,15 +645,16 @@ def react(request): """ discourse_api_key = settings.DISCOURSE_API_KEY - squonk_api_url = settings.SQUONK2_DMAPI_URL - squonk_ui_url = settings.SQUONK2_UI_URL + + # Is the Squonk2 Agent configured? + sq2_rv = _SQ2A.configured() context = {} context['discourse_available'] = 'false' context['squonk_available'] = 'false' if discourse_api_key: context['discourse_available'] = 'true' - if squonk_api_url and squonk_ui_url: + if sq2_rv.success: context['squonk_available'] = 'true' user = request.user @@ -668,8 +673,8 @@ def react(request): # If user is authenticated Squonk can be called then return the Squonk host # so the Frontend can navigate to it context['squonk_ui_url'] = '' - if squonk_api_url and check_squonk_active(request): - context['squonk_ui_url'] = settings.SQUONK2_UI_URL + if sq2_rv.success and check_squonk_active(request): + context['squonk_ui_url'] = _SQ2A.get_ui_url() return render(request, "viewer/react_temp.html", context) @@ -3081,21 +3086,18 @@ def create(self, request): content = {'Only authenticated users can transfer files'} return Response(content, status=status.HTTP_403_FORBIDDEN) - # Can't use this method if the squonk variables are not set! - if not settings.SQUONK2_DMAPI_URL: - content = {'SQUONK2_DMAPI_URL is not set'} + # Can't use this method if the Squonk2 agent is not + sq2 = get_squonk2_agent() + if not sq2.configured(): + content = {'The Squonk2 Agent is not configured'} return Response(content, status=status.HTTP_403_FORBIDDEN) target_id = request.data['target'] target = Target.objects.get(id=target_id) snapshot_id = request.data['snapshot'] - - if 'squonk_project' in request.data: - squonk_project = request.data['squonk_project'] - else: - content = { - 'message': 'A squonk project must be entered'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) + session_project_id = request.data['session_project'] + # The access ID (legacy Project record ID) + access_id = request.data['access'] error, proteins, compounds = check_file_transfer(request) if error: @@ -3123,7 +3125,8 @@ def create(self, request): logger.info('+ target_id=%s', target_id) logger.info('+ snapshot_id=%s', snapshot_id) - logger.info('+ squonk_project=%s', squonk_project) + logger.info('+ session_project_id=%s', session_project_id) + logger.info('+ access_id=%s', access_id) logger.info('+ transfer_root=%s', transfer_root) if job_transfer: @@ -3158,11 +3161,35 @@ def create(self, request): else: # Create new file transfer job + + # This requires a Squonk2 Project (created by the Squonk2Agent). + # It may be an existing project, or it might be a new project. + common_params = CommonParams(user_id=user.id, + access_id=access_id, + target_id=target_id, + session_id=session_project_id) + sq2_rv = _SQ2A.ensure_project(common_params) + if not sq2_rv.success: + msg = f'JobTransfer failed to get a Squonk2 Project' \ + f' for User {user.username}, Access ID {access_id},' \ + f' Target ID {target_id}, and SessionProject ID {session_id}.' \ + f' Returning the message "{sq2_rv.msg}".' \ + ' Cannot continue' + content = {'message': msg} + logger.error(msg) + return Response(content, + status=status.HTTP_404_NOT_FOUND) + # The project UUID is in the response msg (a Squonk2Project object) + squonk2_project = sq2_rv.msg.uuid + job_transfer = JobFileTransfer() job_transfer.user = request.user job_transfer.proteins = [p['code'] for p in proteins] job_transfer.compounds = [c['name'] for c in compounds] - job_transfer.squonk_project = squonk_project + # We should use a foreign key, + # but to avoid migration issues with the existing code + # we continue to use the project UUID string field. + job_transfer.squonk_project = squonk2_project job_transfer.target = Target.objects.get(id=target_id) job_transfer.snapshot = Snapshot.objects.get(id=snapshot_id) @@ -3235,8 +3262,9 @@ def list(self, request): return Response(content, status=status.HTTP_403_FORBIDDEN) # Can't use this method if the squonk variables are not set! - if not settings.SQUONK2_DMAPI_URL: - content = {'SQUONK2_DMAPI_URL is not set'} + sqa_rv = SQ2A.configured() + if sqa_rv.success: + content = {f'The Squonk2 Agent is not configured ({sqa_rv.msg}'} return Response(content, status=status.HTTP_403_FORBIDDEN) job_collection = request.query_params.get('job_collection', None) @@ -3329,8 +3357,10 @@ def create(self, request): return Response(content, status=status.HTTP_403_FORBIDDEN) # Can't use this method if the squonk variables are not set! - if not settings.SQUONK2_DMAPI_URL: - content = {'SQUONK2_DMAPI_URL is not set'} + # Do this for JobRequest and FileTransfer + sq2_rv = _SQ2A.configured + if not sq2_rv.success: + content = {f'The Squonk2 Agent is not configured ({sqa_rv.msg})'} return Response(content, status=status.HTTP_403_FORBIDDEN) try: From 68953ef60705dc973b4f3b7942049d58b432bafd Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 12 Dec 2022 14:05:10 +0000 Subject: [PATCH 017/112] Early /api/projects API method --- api/urls.py | 1 + viewer/serializers.py | 6 ++++++ viewer/views.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/api/urls.py b/api/urls.py index 3ad51f3e..8ce84069 100644 --- a/api/urls.py +++ b/api/urls.py @@ -15,6 +15,7 @@ router.register(r"compounds", viewer_views.CompoundView) router.register(r"targets", viewer_views.TargetView, "targets") router.register(r"proteins", viewer_views.ProteinView) +router.register(r"projects", viewer_views.ProjectView) router.register(r"session-projects", viewer_views.SessionProjectsView) router.register(r"snapshots", viewer_views.SnapshotsView) router.register(r"action-type", viewer_views.ActionTypeView) diff --git a/viewer/serializers.py b/viewer/serializers.py index 3f8d732e..6e606a47 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -122,6 +122,12 @@ def template_protein(obj): return "NOT AVAILABLE" +class ProjectSerializer(serializers.ModelSerializer): + class Meta: + model = Project + fields = '__all__' + + class TargetSerializer(serializers.ModelSerializer): template_protein = serializers.SerializerMethodField() zip_archive = serializers.SerializerMethodField() diff --git a/viewer/views.py b/viewer/views.py index ff56c187..00c0094d 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -407,6 +407,36 @@ class ProteinPDBBoundInfoView(ISpyBSafeQuerySet): filterset_fields = ("code", "target_id", "target_id__title", "prot_type") +class ProjectView(ISpyBSafeQuerySet): + """ DjagnoRF view to retrieve info about projects + + Methods + ------- + url: + api/projects + queryset: + `viewer.models.Project.objects.filter()` + returns: JSON + - id: id of the target object + - title: name of the target + - init_date: The date the Project was created + + example output: + + .. code-block:: javascript + + "results": [ + { + "id": 2, + "title": "Mpro", + "init_date": "" + } + ] + + """ + queryset = Project.objects.filter() + serializer_class = ProjectSerializer + class TargetView(ISpyBSafeQuerySet): """ DjagnoRF view to retrieve info about targets From 941dac9d67962db49dbedf86a82d29d6bf016217 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 4 Jan 2023 11:59:54 +0000 Subject: [PATCH 018/112] Use of new Sq Python client and shortuuid package --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8d0249f2..1a1d19cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.3.0 +im-squonk2-client==1.15.1 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 @@ -101,6 +101,7 @@ Rx==1.6.1 scandir==1.10.0 SecretStorage==3.1.2 sentry-sdk==0.16.3 +shortuuid==1.0.11 simplegeneric==0.8.1 simplejson==3.17.2 singledispatch==3.4.0.3 From de12bfcd10a5ceb41ef634ffb2b64bfd47f28db4 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 4 Jan 2023 12:00:08 +0000 Subject: [PATCH 019/112] Doc typo --- viewer/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/viewer/models.py b/viewer/models.py index ba4dfdb7..c84e946f 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -1188,7 +1188,7 @@ class JobRequest(models.Model): # For us this will contain a 'task_id', 'instance_id' and 'callback_token'. # The content will be a list with index '0' that's the value of the DmApiRv # 'success' variable and, at index '1', the original response message json(). - # The Job callback token will be squonk_job_info[0]['callback_token'] + # The Job callback token will be squonk_job_info[1]['callback_token'] squonk_job_info = models.JSONField(encoder=DjangoJSONEncoder, null=True) # 'squonk_url_ext' is a Squonk UI URL to obtain information about the # running instance. It's essentially the Squonk URL with the instance ID appended. @@ -1202,5 +1202,3 @@ class JobRequest(models.Model): class Meta: db_table = 'viewer_jobrequest' # End of Squonk Job Tables - - From a74fa575eca66a4c96508c14fee120c232c77731 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 4 Jan 2023 12:00:41 +0000 Subject: [PATCH 020/112] Attempt to fix callback race-condition --- viewer/squonk_job_file_upload.py | 43 ++++++++++++++++++++------------ viewer/squonk_job_request.py | 41 +++++++++++++++++++++++++----- viewer/views.py | 11 +++++++- 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/viewer/squonk_job_file_upload.py b/viewer/squonk_job_file_upload.py index 37ac7d34..467fcb18 100644 --- a/viewer/squonk_job_file_upload.py +++ b/viewer/squonk_job_file_upload.py @@ -138,17 +138,27 @@ def process_compound_set_file(jr_id, logger.info('Processing job compound file (%s)...', jr_id) + logger.info("Squonk transition_time='%s'", transition_time) + logger.info("Squonk job_output_path='%s'", job_output_path) + logger.info("Squonk job_output_filename='%s'", job_output_filename) + jr = JobRequest.objects.get(id=jr_id) - # The callback token (required to make Squonk API calls from the callback) - # and instance ID are in JobRequest. + # The callback token is required to make Squonk API calls from the callback context jr_job_info_msg = jr.squonk_job_info[1] - token = jr_job_info_msg.get('callback_token') - instance_id = jr_job_info_msg.get('instance_id') - logger.info("Squonk API token=%s", token) - logger.info("Squonk API instance_id=%s", instance_id) - - logger.info("Expecting Squonk path='%s'", job_output_path) + callback_token = jr_job_info_msg.get('callback_token') + logger.info("Squonk API callback_token=%s", callback_token) + + # The Squonk instance that ran the Job can be found + # at the end of the jr.squonk_url_ext field. + instance_id = '' + if jr.squonk_url_ext: + i_id = jr.squonk_url_ext.split('/')[-1] + if i_id.startswith('instance-'): + instance_id = i_id + logger.info("Squonk instance_id='%s'", instance_id) + else: + logger.warning("jr.squonk_url_ext does not contain an instance ID") # Do we need to create the upload path? # This is used for this 'job' and is removed when the upload is complete @@ -182,7 +192,7 @@ def process_compound_set_file(jr_id, logger.info("Expecting Squonk job_output_param_filename='%s'...", job_output_param_filename) result = DmApi.get_unmanaged_project_file_with_token( - token=token, + token=callback_token, project_id=jr.squonk_project, project_path=job_output_path, project_file=job_output_param_filename, @@ -197,7 +207,7 @@ def process_compound_set_file(jr_id, logger.info("Expecting Squonk job_output_filename='%s'...", job_output_filename) result = DmApi.get_unmanaged_project_file_with_token( - token=token, + token=callback_token, project_id=jr.squonk_project, project_path=job_output_path, project_file=job_output_filename, @@ -209,12 +219,13 @@ def process_compound_set_file(jr_id, else: # Both files pulled back. got_all_files = True - # Delete the callback token, which is no-longer needed. - # Don't care if this fails - the token will expire automatically - # after a period of time. -# _ = DmApi.delete_instance_token( -# instance_id=instance_id, -# token=token) +# if instance_id: +# # Delete the callback token, which is no-longer needed. +# # Don't care if this fails - the token will expire automatically +# # after a period of time. +# _ = DmApi.delete_instance_token( +# instance_id=instance_id, +# token=callback_token) if not got_all_files: logger.warning('Not processing. Either %s or %s is missing', diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index f0484f97..21c29ddc 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -8,6 +8,8 @@ import logging import datetime +import shortuuid + from django.conf import settings from squonk2.dm_api import DmApi @@ -134,14 +136,46 @@ def create_squonk_job(request): # Used for identifying the run, set to the username + date. job_name = job_request.user.username + '-' + datetime.date.today().strftime('%Y-%m-%d') + # Create a callback token + # that we can use on the job from our callback context. + # It is required to be a shortuuid of 22 characters using the default character set + callback_token = shortuuid.uuid() + logger.info('+ job_name=%s', job_name) logger.info('+ callback_url=%s', callback_url) + logger.info('+ callback_token=%s', callback_token) + + # Dry-run the Job execution (this provides us with the 'command', which is + # placed in the JobRecord's squonk_job_info). + logger.info('+ Calling DmApi.dry_run_job_instance(%s)', job_name) + result = DmApi.dry_run_job_instance(auth_token, + project_id=job_request.squonk_project, + name=job_name, + callback_url=callback_url, + callback_token=callback_token, + specification=json.loads(squonk_job_spec)) + logger.debug(result) + + if not result.success: + logger.warning('+ dry_run_job_instance(%s) result=%s', job_name, result) + logger.error('+ FAILED (configuration problem) (%s)', job_name) + job_request.delete() + raise ValueError(result.msg) + + # We can now commit the JobRequest record so that it's ready + # for use by any callbacks. The 'result' will contain the callback token + # and the Job's decoded command (that will be interrogated when the Job s complete) + job_request.squonk_job_info = result + job_request.job_start_datetime = datetime.datetime.utcnow() + job_request.save() + + # Now start the job 'for real'... logger.info('+ Calling DmApi.start_job_instance(%s)', job_name) result = DmApi.start_job_instance(auth_token, project_id=job_request.squonk_project, name=job_name, callback_url=callback_url, - generate_callback_token=True, + callback_token=callback_token, specification=json.loads(squonk_job_spec), timeout_s=8) logger.debug(result) @@ -152,11 +186,6 @@ def create_squonk_job(request): job_request.delete() raise ValueError(result.msg) - job_request.squonk_job_info = result - job_request.squonk_url_ext = settings.SQUONK2_INSTANCE_API + str(result.msg['instance_id']) - job_request.job_start_datetime = datetime.datetime.utcnow() - job_request.save() - job_instance_id = result.msg['instance_id'] job_task_id = result.msg['task_id'] logger.info('+ SUCCESS. Job "%s" started (job_instance_id=%s job_task_id=%s)', diff --git a/viewer/views.py b/viewer/views.py index 0d392787..2edec24e 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3422,6 +3422,15 @@ def update(self, request, code=None): code, status) return HttpResponse(status=204) + # This is now a chance to set the squonk_url_ext using the instance ID + # present in the callback (if it's not already set). The instance is + # placed at the end of the string, and is expected to be found + # by process_compound_set_file() whcih loads the output file back + # into Fragalysis + if not jr.squonk_url_ext: + jr.squonk_url_ext = settings.SQUONK2_INSTANCE_API + str(request.data['instance_id']) + logger.info("Setting jr.squonk_url_ext='%s'", jr.squonk_url_ext) + # Update the state transition time, # assuming UTC. transition_time = request.data.get('state_transition_time') @@ -3447,7 +3456,7 @@ def update(self, request, code=None): logger.info('Setting job FINISH datetime (%s)', transition_time) jr.job_finish_datetime = transition_time_utc - # Save - before going further. + # Save the JobRequest record before going further. jr.save() if status != 'SUCCESS': From 9774d9f8b8a303340100e76427393a234080826a Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 5 Jan 2023 15:15:09 +0000 Subject: [PATCH 021/112] Attempt to fix agent reloading --- viewer/serializers.py | 6 ------ viewer/squonk2_agent.py | 18 +++++++----------- viewer/views.py | 34 +++++++++++++++++++++++----------- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/viewer/serializers.py b/viewer/serializers.py index 6e606a47..3f8d732e 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -122,12 +122,6 @@ def template_protein(obj): return "NOT AVAILABLE" -class ProjectSerializer(serializers.ModelSerializer): - class Meta: - model = Project - fields = '__all__' - - class TargetSerializer(serializers.ModelSerializer): template_protein = serializers.SerializerMethodField() zip_archive = serializers.SerializerMethodField() diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index c1b0370e..dec2bc7e 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -21,6 +21,7 @@ from requests import Response from wrapt import synchronized +from fragalysis.settings import SQUONK2_AGENT_SINGLETON from viewer.models import User, SessionProject, Project from viewer.models import Squonk2Project, Squonk2Org, Squonk2Unit @@ -769,21 +770,16 @@ def ensure_project(self, params: CommonParams) -> Squonk2AgentRv: return rv_u -# The global (singleton). -# This acts as out sole singleton, -# created and returned from `get_squonk2_agent()' -_SQUONK2_AGENT: Optional[Squonk2Agent] = None - def get_squonk2_agent() -> Squonk2Agent: - """Returns a 'singleton'. + """Returns a 'singleton' (reference stored in settings.py). """ - global _SQUONK2_AGENT # pylint: disable=global-statement + global SQUONK2_AGENT_SINGLETON # pylint: disable=global-statement - if _SQUONK2_AGENT: - return _SQUONK2_AGENT + if SQUONK2_AGENT_SINGLETON: + return SQUONK2_AGENT_SINGLETON _LOGGER.debug("Creating new Squonk2Agent...") - _SQUONK2_AGENT = Squonk2Agent() + SQUONK2_AGENT_SINGLETON = Squonk2Agent() _LOGGER.debug("Created") - return _SQUONK2_AGENT + return SQUONK2_AGENT_SINGLETON diff --git a/viewer/views.py b/viewer/views.py index 00c0094d..d7f67153 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -417,7 +417,7 @@ class ProjectView(ISpyBSafeQuerySet): queryset: `viewer.models.Project.objects.filter()` returns: JSON - - id: id of the target object + - id: id of the project object - title: name of the target - init_date: The date the Project was created @@ -434,8 +434,16 @@ class ProjectView(ISpyBSafeQuerySet): ] """ - queryset = Project.objects.filter() - serializer_class = ProjectSerializer + def get(self, request): + # Get the proposals available to this user. + # We return the corresponding Project records + user = request.user + proposals = self.get_proposals_for_user() + logger.info("User %s proposals %s", user.username, proposals) + + response_data = {} + return JsonResponse(response_data) + class TargetView(ISpyBSafeQuerySet): """ DjagnoRF view to retrieve info about targets @@ -676,19 +684,24 @@ def react(request): discourse_api_key = settings.DISCOURSE_API_KEY + context = {} + # Is the Squonk2 Agent configured? + logger.info("Checking whether Squonk2 is configured...") sq2_rv = _SQ2A.configured() + if sq2_rv.success: + logger.info("Squonk2 is configured") + context['squonk_available'] = 'true' + else: + logger.info("Squonk2 is NOT configured") + context['squonk_available'] = 'false' - context = {} - context['discourse_available'] = 'false' - context['squonk_available'] = 'false' if discourse_api_key: context['discourse_available'] = 'true' - if sq2_rv.success: - context['squonk_available'] = 'true' + else: + context['discourse_available'] = 'false' user = request.user - if user.is_authenticated: context['discourse_host'] = '' context['user_present_on_discourse'] = 'false' @@ -3117,8 +3130,7 @@ def create(self, request): return Response(content, status=status.HTTP_403_FORBIDDEN) # Can't use this method if the Squonk2 agent is not - sq2 = get_squonk2_agent() - if not sq2.configured(): + if not _SQ2A.configured(): content = {'The Squonk2 Agent is not configured'} return Response(content, status=status.HTTP_403_FORBIDDEN) From e1ca199852e29aa658d5de502c1fa7ee76bea5d2 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 9 Jan 2023 12:14:16 +0000 Subject: [PATCH 022/112] Minor fixes for project API endpoint --- api/urls.py | 2 +- docker-compose.yml | 2 ++ viewer/views.py | 11 +++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/api/urls.py b/api/urls.py index 8ce84069..04a13d6f 100644 --- a/api/urls.py +++ b/api/urls.py @@ -15,7 +15,7 @@ router.register(r"compounds", viewer_views.CompoundView) router.register(r"targets", viewer_views.TargetView, "targets") router.register(r"proteins", viewer_views.ProteinView) -router.register(r"projects", viewer_views.ProjectView) +router.register(r"projects", viewer_views.ProjectView, "projects") router.register(r"session-projects", viewer_views.SessionProjectsView) router.register(r"snapshots", viewer_views.SnapshotsView) router.register(r"action-type", viewer_views.ActionTypeView) diff --git a/docker-compose.yml b/docker-compose.yml index e00dbd7f..f3f61690 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ # docker-compose up -d # Then enter the stack container with: - # docker-compose exec stack bash +# Where you should then find the back end at http://localhost:8080/api/ +# and logs in /code/logs/backend.log version: '3' diff --git a/viewer/views.py b/viewer/views.py index d7f67153..fe32d471 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -40,6 +40,7 @@ from viewer.models import ( Molecule, Protein, + Project, Compound, Target, ActionType, @@ -49,10 +50,8 @@ SnapshotActions, ComputedMolecule, ComputedSet, - CSetKeys, NumericalScoreValues, ScoreDescription, - File, TagCategory, TextScoreValues, MoleculeTag, @@ -113,7 +112,6 @@ SnapshotReadSerializer, SnapshotWriteSerializer, SnapshotActionsSerializer, - FileSerializer, ComputedSetSerializer, ComputedMoleculeSerializer, NumericalScoreSerializer, @@ -132,7 +130,8 @@ JobRequestReadSerializer, JobRequestWriteSerializer, JobCallBackReadSerializer, - JobCallBackWriteSerializer + JobCallBackWriteSerializer, + ProjectSerializer, ) logger = logging.getLogger(__name__) @@ -434,6 +433,10 @@ class ProjectView(ISpyBSafeQuerySet): ] """ + queryset = Project.objects.filter() + serializer_class = ProjectSerializer + filter_permissions = "target" + def get(self, request): # Get the proposals available to this user. # We return the corresponding Project records From bd207221a9068cba75a63648f50d5bef290bff68 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 9 Jan 2023 15:35:56 +0000 Subject: [PATCH 023/112] No send code/logs to stdout --- launch-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch-stack.sh b/launch-stack.sh index 1a7559c4..ef1a29d3 100755 --- a/launch-stack.sh +++ b/launch-stack.sh @@ -22,7 +22,7 @@ printf "$script" | python manage.py shell touch /srv/logs/gunicorn.log touch /srv/logs/access.log touch /code/logs/logfile.log -tail -n 0 -f /srv/logs/*.log & +tail -n 0 -f /code/logs/*.log & echo "Starting Gunicorn...." cd /code gunicorn fragalysis.wsgi:application \ From b16782670b0eaf36d14785459ea603f5c465e4ca Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 9 Jan 2023 15:38:32 +0000 Subject: [PATCH 024/112] Better project serializer --- viewer/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viewer/serializers.py b/viewer/serializers.py index 3f8d732e..31b1567d 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -287,7 +287,7 @@ class ProjectSerializer(serializers.ModelSerializer): class Meta: model = Project - fields = ("id", "title") + fields = ("id", "title", "init_date") class MolImageSerialzier(serializers.ModelSerializer): From 03411dfab15bcf7346a92fd95b078ff1ff054025 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 9 Jan 2023 15:39:05 +0000 Subject: [PATCH 025/112] Better urls --- api/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/urls.py b/api/urls.py index 04a13d6f..8ce84069 100644 --- a/api/urls.py +++ b/api/urls.py @@ -15,7 +15,7 @@ router.register(r"compounds", viewer_views.CompoundView) router.register(r"targets", viewer_views.TargetView, "targets") router.register(r"proteins", viewer_views.ProteinView) -router.register(r"projects", viewer_views.ProjectView, "projects") +router.register(r"projects", viewer_views.ProjectView) router.register(r"session-projects", viewer_views.SessionProjectsView) router.register(r"snapshots", viewer_views.SnapshotsView) router.register(r"action-type", viewer_views.ActionTypeView) From 7a7467bde7c9d9b0d6d2e1b452a48f64dffbb2bc Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 9 Jan 2023 15:39:31 +0000 Subject: [PATCH 026/112] Fix for Project view --- viewer/views.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/viewer/views.py b/viewer/views.py index fe32d471..e72c14f1 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -427,25 +427,16 @@ class ProjectView(ISpyBSafeQuerySet): "results": [ { "id": 2, - "title": "Mpro", - "init_date": "" + "title": "lb27156", + "init_date": "2023-01-09T15:00:00Z" } ] """ queryset = Project.objects.filter() serializer_class = ProjectSerializer - filter_permissions = "target" - - def get(self, request): - # Get the proposals available to this user. - # We return the corresponding Project records - user = request.user - proposals = self.get_proposals_for_user() - logger.info("User %s proposals %s", user.username, proposals) - - response_data = {} - return JsonResponse(response_data) + # Special case - Project filter permissions is blank. + filter_permissions = "" class TargetView(ISpyBSafeQuerySet): From 1e11eec5479a376af6854e3be16e189269732d47 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 9 Jan 2023 15:39:53 +0000 Subject: [PATCH 027/112] Fix for Project query --- api/security.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/api/security.py b/api/security.py index a6ca7466..0bda18d4 100644 --- a/api/security.py +++ b/api/security.py @@ -1,3 +1,4 @@ +import logging import os import time from wsgiref.util import FileWrapper @@ -10,6 +11,8 @@ from viewer.models import Project +logger = logging.getLogger(__name__) + USER_LIST_DICT = {} connector = os.environ.get('SECURITY_CONNECTOR', 'ispyb') @@ -86,12 +89,15 @@ def get_queryset(self): """ Optionally restricts the returned purchases to a given proposals """ + logger.info("A") # The list of proposals this user can have proposal_list = self.get_proposals_for_user() # Add in the ones everyone has access to proposal_list.extend(self.get_open_proposals()) - # Must have a directy foreign key (project_id) for it to work + logger.info("A proposal_list=%s", proposal_list) + # Must have a directly foreign key (project_id) for it to work filter_dict = self.get_filter_dict(proposal_list) + logger.info("A filter_dict=%s", filter_dict) return self.queryset.filter(**filter_dict).distinct() def get_open_proposals(self): @@ -99,6 +105,7 @@ def get_open_proposals(self): Returns the list of proposals anybody can access :return: """ + logger.info("B") if os.environ.get("TEST_SECURITY_FLAG", False): return ["OPEN", "private_dummy_project"] else: @@ -106,6 +113,7 @@ def get_open_proposals(self): def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user + logger.info("C") if user.pk is None: return [] else: @@ -115,6 +123,8 @@ def get_proposals_for_user_from_django(self, user): def needs_updating(self, user): global USER_LIST_DICT + + logger.info("D") update_window = 3600 if user.username not in USER_LIST_DICT: USER_LIST_DICT[user.username] = {"RESULTS": [], "TIMESTAMP": 0} @@ -125,6 +135,8 @@ def needs_updating(self, user): return False def run_query_with_connector(self, conn, user): + + logger.info("E") core = conn.core try: rs = core.retrieve_sessions_for_person_login(user.username) @@ -139,6 +151,8 @@ def run_query_with_connector(self, conn, user): def get_proposals_for_user_from_ispyb(self, user): # First check if it's updated in the past 1 hour global USER_LIST_DICT + + logger.info("F") if self.needs_updating(user): conn = None if connector == 'ispyb': @@ -164,6 +178,8 @@ def get_proposals_for_user_from_ispyb(self, user): return USER_LIST_DICT[user.username]["RESULTS"] def get_proposals_for_user(self): + + logger.info("G") user = self.request.user ispyb_user = os.environ.get("ISPYB_USER") if ispyb_user: @@ -175,7 +191,13 @@ def get_proposals_for_user(self): return self.get_proposals_for_user_from_django(user) def get_filter_dict(self, proposal_list): - return {self.filter_permissions + "__title__in": proposal_list} + if self.filter_permissions: + return {self.filter_permissions + "__title__in": proposal_list} + else: + # No filter permission? + # Assume this is QuerySet is used for the Project model. + # Added during 937 development (Access Control). + return {"title__in": proposal_list} class ISpyBSafeStaticFiles: From 960ad157e5c887930425cb82267740bb2321613f Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 9 Jan 2023 15:40:56 +0000 Subject: [PATCH 028/112] Removed logging --- api/security.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/api/security.py b/api/security.py index 0bda18d4..6d6d790f 100644 --- a/api/security.py +++ b/api/security.py @@ -1,4 +1,3 @@ -import logging import os import time from wsgiref.util import FileWrapper @@ -11,8 +10,6 @@ from viewer.models import Project -logger = logging.getLogger(__name__) - USER_LIST_DICT = {} connector = os.environ.get('SECURITY_CONNECTOR', 'ispyb') @@ -89,15 +86,12 @@ def get_queryset(self): """ Optionally restricts the returned purchases to a given proposals """ - logger.info("A") # The list of proposals this user can have proposal_list = self.get_proposals_for_user() # Add in the ones everyone has access to proposal_list.extend(self.get_open_proposals()) - logger.info("A proposal_list=%s", proposal_list) # Must have a directly foreign key (project_id) for it to work filter_dict = self.get_filter_dict(proposal_list) - logger.info("A filter_dict=%s", filter_dict) return self.queryset.filter(**filter_dict).distinct() def get_open_proposals(self): @@ -105,7 +99,6 @@ def get_open_proposals(self): Returns the list of proposals anybody can access :return: """ - logger.info("B") if os.environ.get("TEST_SECURITY_FLAG", False): return ["OPEN", "private_dummy_project"] else: @@ -113,7 +106,6 @@ def get_open_proposals(self): def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user - logger.info("C") if user.pk is None: return [] else: @@ -124,7 +116,6 @@ def get_proposals_for_user_from_django(self, user): def needs_updating(self, user): global USER_LIST_DICT - logger.info("D") update_window = 3600 if user.username not in USER_LIST_DICT: USER_LIST_DICT[user.username] = {"RESULTS": [], "TIMESTAMP": 0} @@ -136,7 +127,6 @@ def needs_updating(self, user): def run_query_with_connector(self, conn, user): - logger.info("E") core = conn.core try: rs = core.retrieve_sessions_for_person_login(user.username) @@ -152,7 +142,6 @@ def get_proposals_for_user_from_ispyb(self, user): # First check if it's updated in the past 1 hour global USER_LIST_DICT - logger.info("F") if self.needs_updating(user): conn = None if connector == 'ispyb': @@ -179,7 +168,6 @@ def get_proposals_for_user_from_ispyb(self, user): def get_proposals_for_user(self): - logger.info("G") user = self.request.user ispyb_user = os.environ.get("ISPYB_USER") if ispyb_user: From 587ea9d015568ebe11ac651d8a0edea58db9f60e Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 10 Jan 2023 10:24:47 +0000 Subject: [PATCH 029/112] Project serialiser now masquerades new record content - Fixed squonk2agent typos - pre-existing Serialiser (zer) typos --- viewer/serializers.py | 28 ++++++++++++++++++++++------ viewer/squonk2_agent.py | 15 ++++++++------- viewer/views.py | 25 +++++++++++++------------ 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/viewer/serializers.py b/viewer/serializers.py index 31b1567d..245d2844 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -285,12 +285,28 @@ class Meta: class ProjectSerializer(serializers.ModelSerializer): + # Field name translation (prior to refactoring the Model) + # 'tas' is the new name for 'title' + target_access_string = serializers.SerializerMethodField('get_tas') + # 'authority' is the (as yet to be implemented) origin of the TAS + # For now this is fixed at "DIAMOND-ISPYB" + authority = serializers.SerializerMethodField('get_authority') + class Meta: model = Project - fields = ("id", "title", "init_date") + fields = ("id", "target_access_string", "init_date", "authority") + + def get_tas(self, instance): + return instance.title + + def get_authority(self, instance): + # Don't actually need the instance here. + # We return a hard-coded string. + del instance + return "DIAMOND-ISPYB" -class MolImageSerialzier(serializers.ModelSerializer): +class MolImageSerializer(serializers.ModelSerializer): mol_image = serializers.SerializerMethodField() @@ -311,7 +327,7 @@ class Meta: fields = ("id", "mol_image") -class CmpdImageSerialzier(serializers.ModelSerializer): +class CmpdImageSerializer(serializers.ModelSerializer): cmpd_image = serializers.SerializerMethodField() @@ -323,7 +339,7 @@ class Meta: fields = ("id", "cmpd_image") -class ProtMapInfoSerialzer(serializers.ModelSerializer): +class ProtMapInfoSerializer(serializers.ModelSerializer): map_data = serializers.SerializerMethodField() @@ -338,7 +354,7 @@ class Meta: fields = ("id", "map_data", "prot_type") -class ProtPDBInfoSerialzer(serializers.ModelSerializer): +class ProtPDBInfoSerializer(serializers.ModelSerializer): pdb_data = serializers.SerializerMethodField() @@ -351,7 +367,7 @@ class Meta: fields = ("id", "pdb_data", "prot_type") -class ProtPDBBoundInfoSerialzer(serializers.ModelSerializer): +class ProtPDBBoundInfoSerializer(serializers.ModelSerializer): bound_pdb_data = serializers.SerializerMethodField() diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index dec2bc7e..6a757cd0 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -21,7 +21,6 @@ from requests import Response from wrapt import synchronized -from fragalysis.settings import SQUONK2_AGENT_SINGLETON from viewer.models import User, SessionProject, Project from viewer.models import Squonk2Project, Squonk2Org, Squonk2Unit @@ -770,16 +769,18 @@ def ensure_project(self, params: CommonParams) -> Squonk2AgentRv: return rv_u +# A placeholder for the Agent object +_AGENT_SINGLETON: Optional[Squonk2Agent] = None def get_squonk2_agent() -> Squonk2Agent: - """Returns a 'singleton' (reference stored in settings.py). + """Returns a 'singleton'. """ - global SQUONK2_AGENT_SINGLETON # pylint: disable=global-statement + global _AGENT_SINGLETON # pylint: disable=global-statement - if SQUONK2_AGENT_SINGLETON: - return SQUONK2_AGENT_SINGLETON + if _AGENT_SINGLETON: + return _AGENT_SINGLETON _LOGGER.debug("Creating new Squonk2Agent...") - SQUONK2_AGENT_SINGLETON = Squonk2Agent() + _AGENT_SINGLETON = Squonk2Agent() _LOGGER.debug("Created") - return SQUONK2_AGENT_SINGLETON + return _AGENT_SINGLETON diff --git a/viewer/views.py b/viewer/views.py index e72c14f1..5cb4316c 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -98,11 +98,11 @@ ProteinSerializer, CompoundSerializer, TargetSerializer, - MolImageSerialzier, - CmpdImageSerialzier, - ProtMapInfoSerialzer, - ProtPDBInfoSerialzer, - ProtPDBBoundInfoSerialzer, + MolImageSerializer, + CmpdImageSerializer, + ProtMapInfoSerializer, + ProtPDBInfoSerializer, + ProtPDBBoundInfoSerializer, VectorsSerializer, GraphSerializer, ActionTypeSerializer, @@ -277,7 +277,7 @@ class MolImageView(ISpyBSafeQuerySet): "mol_image": " Date: Tue, 10 Jan 2023 12:16:36 +0000 Subject: [PATCH 030/112] A fix for squonk_url_ext --- viewer/squonk_job_request.py | 9 +++++++-- viewer/utils.py | 7 ++++++- viewer/views.py | 9 +++++---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index 21c29ddc..9554d0dc 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -17,7 +17,7 @@ Snapshot, JobRequest, JobFileTransfer ) -from viewer.utils import get_https_host +from viewer.utils import create_squonk_job_request_url, get_https_host logger = logging.getLogger(__name__) @@ -191,4 +191,9 @@ def create_squonk_job(request): logger.info('+ SUCCESS. Job "%s" started (job_instance_id=%s job_task_id=%s)', job_name, job_instance_id, job_task_id) - return job_request.id, job_request.squonk_url_ext + # Manufacture the Squonk URL (actually set in the callback) + # so the front-end can use it immediately (we cannot set the JobRequest + # `squonk_url_ext` here as it introduces a race-condition with the callback logic). + squonk_url_ext = create_squonk_job_request_url(job_instance_id) + + return job_request.id, squonk_url_ext diff --git a/viewer/utils.py b/viewer/utils.py index 34199b83..b01d5c88 100644 --- a/viewer/utils.py +++ b/viewer/utils.py @@ -16,6 +16,12 @@ SDF_VERSION = 'ver_1.2' +def create_squonk_job_request_url(instance_id): + """Creates the Squonk Instance API url from an instance ID (UUID). + """ + return settings.SQUONK2_INSTANCE_API + str(instance_id) + + def create_media_sub_directory(sub_directory): """Creates a directory (or directories) in the MEDIA directory, returning the full path. @@ -133,4 +139,3 @@ def get_https_host(request): Note that this link will not work on local """ return settings.SECURE_PROXY_SSL_HEADER[1] + '://' + request.get_host() - diff --git a/viewer/views.py b/viewer/views.py index 2edec24e..c02d0a85 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -36,6 +36,7 @@ from api.security import ISpyBSafeQuerySet from api.utils import get_params, get_highlighted_diffs +from viewer.utils import create_squonk_job_request_url from viewer.models import ( Molecule, @@ -3422,15 +3423,15 @@ def update(self, request, code=None): code, status) return HttpResponse(status=204) - # This is now a chance to set the squonk_url_ext using the instance ID + # This is now a chance to safely set the squonk_url_ext using the instance ID # present in the callback (if it's not already set). The instance is # placed at the end of the string, and is expected to be found - # by process_compound_set_file() whcih loads the output file back + # by process_compound_set_file() which loads the output file back # into Fragalysis if not jr.squonk_url_ext: - jr.squonk_url_ext = settings.SQUONK2_INSTANCE_API + str(request.data['instance_id']) + jr.squonk_url_ext = create_squonk_job_request_url(request.data['instance_id']) logger.info("Setting jr.squonk_url_ext='%s'", jr.squonk_url_ext) - + # Update the state transition time, # assuming UTC. transition_time = request.data.get('state_transition_time') From 12925d09d905d58d436e64a008b57e31d7f36222 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 10 Jan 2023 15:20:53 +0000 Subject: [PATCH 031/112] Better initialisation of squonk2 agent --- viewer/squonk2_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 6a757cd0..ae39d6e3 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -108,7 +108,7 @@ def __init__(self): self.__CFG_SQUONK2_PRODUCT_FLAVOUR: Optional[str] =\ os.environ.get('SQUONK2_PRODUCT_FLAVOUR') self.__CFG_SQUONK2_SLUG: Optional[str] =\ - os.environ.get('SQUONK2_SLUG')[:_MAX_SLUG_LENGTH] + os.environ.get('SQUONK2_SLUG', '')[:_MAX_SLUG_LENGTH] self.__CFG_SQUONK2_ORG_OWNER: Optional[str] =\ os.environ.get('SQUONK2_ORG_OWNER') self.__CFG_SQUONK2_ORG_OWNER_PASSWORD: Optional[str] =\ From e077eab56fad42f64f83ebac599aeeed65b59705 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 11 Jan 2023 14:39:52 +0000 Subject: [PATCH 032/112] Removed unused variable --- viewer/squonk2_agent.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index ae39d6e3..84457b46 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -51,12 +51,12 @@ SendParams: namedtuple = namedtuple("Send", ["common", "snapshot_id"]) - _SUPPORTED_PRODUCT_FLAVOURS: List[str] = ["BRONZE", "SILVER", "GOLD"] # Squonk2 Have defined limits - assumed here. # verify with your Squonk2 installation. -_MAX_SQ2_NAME_LENGTH: int = 80 +# How long are Squonk2 'names'? +_SQ2_MAX_NAME_LENGTH: int = 80 # A slug used for names this Fragalysis will create # and a prefix string. So Squonk2 objects will be called "Fragalysis {slug}" @@ -66,9 +66,6 @@ # Built-in _SQ2_PRODUCT_TYPE: str = 'DATA_MANAGER_PROJECT_TIER_SUBSCRIPTION' -# How long are Squonk2 'names'? -_SQ2_MAX_NAME_LENGTH: int = 80 - # True if the code's in Test Mode _TEST_MODE: bool = True From 50565a4e8eece69dad2150b7597a5335d324f01d Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 12 Jan 2023 11:18:11 +0000 Subject: [PATCH 033/112] Adds 'open-to-public' property to Project record --- .../0026_add_project_open_to_public_field.py | 18 ++++++++++++++++++ viewer/models.py | 3 +++ viewer/serializers.py | 2 +- viewer/views.py | 3 ++- 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 viewer/migrations/0026_add_project_open_to_public_field.py diff --git a/viewer/migrations/0026_add_project_open_to_public_field.py b/viewer/migrations/0026_add_project_open_to_public_field.py new file mode 100644 index 00000000..3e0a84ec --- /dev/null +++ b/viewer/migrations/0026_add_project_open_to_public_field.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2023-01-12 11:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0025_squonk2org_squonk2project_squonk2unit'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='open_to_public', + field=models.BooleanField(default=False), + ), + ] diff --git a/viewer/models.py b/viewer/models.py index 7ffaa542..7a29d4c9 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -22,6 +22,8 @@ class Project(models.Model): The date the project was initiated (autofield) user_id: ManyToManyField Links to the User model + open_to_public: BooleanField + True if open to the Public """ # The title of the project_id -> userdefined title = models.CharField(max_length=200, unique=True) @@ -29,6 +31,7 @@ class Project(models.Model): init_date = models.DateTimeField(auto_now_add=True) # The users it's related to user_id = models.ManyToManyField(User) + open_to_public = models.BooleanField(default=False) class Target(models.Model): diff --git a/viewer/serializers.py b/viewer/serializers.py index 245d2844..96042098 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -294,7 +294,7 @@ class ProjectSerializer(serializers.ModelSerializer): class Meta: model = Project - fields = ("id", "target_access_string", "init_date", "authority") + fields = ("id", "target_access_string", "init_date", "authority", "open_to_public") def get_tas(self, instance): return instance.title diff --git a/viewer/views.py b/viewer/views.py index 5dd65e4f..03fe0460 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -430,7 +430,8 @@ class ProjectView(ISpyBSafeQuerySet): "id": 2, "target_access_string": "lb27156-1", "init_date": "2023-01-09T15:00:00Z", - "authority": "DIAMOND-ISPYB" + "authority": "DIAMOND-ISPYB", + "open_to_public": false } ] From 27e01725c3e1e4ad43dd7cedf8b5566a4a05945d Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 12 Jan 2023 12:53:29 +0000 Subject: [PATCH 034/112] Removed unused get_open_proposals --- api/security.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/api/security.py b/api/security.py index 6d6d790f..91171a54 100644 --- a/api/security.py +++ b/api/security.py @@ -88,22 +88,10 @@ def get_queryset(self): """ # The list of proposals this user can have proposal_list = self.get_proposals_for_user() - # Add in the ones everyone has access to - proposal_list.extend(self.get_open_proposals()) # Must have a directly foreign key (project_id) for it to work filter_dict = self.get_filter_dict(proposal_list) return self.queryset.filter(**filter_dict).distinct() - def get_open_proposals(self): - """ - Returns the list of proposals anybody can access - :return: - """ - if os.environ.get("TEST_SECURITY_FLAG", False): - return ["OPEN", "private_dummy_project"] - else: - return ["OPEN", "lb27156"] - def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user if user.pk is None: From c9ed6285fbcb21940da029fdec83ddfe3236992f Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 12 Jan 2023 13:01:27 +0000 Subject: [PATCH 035/112] Revert security change --- api/security.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/api/security.py b/api/security.py index 91171a54..6d6d790f 100644 --- a/api/security.py +++ b/api/security.py @@ -88,10 +88,22 @@ def get_queryset(self): """ # The list of proposals this user can have proposal_list = self.get_proposals_for_user() + # Add in the ones everyone has access to + proposal_list.extend(self.get_open_proposals()) # Must have a directly foreign key (project_id) for it to work filter_dict = self.get_filter_dict(proposal_list) return self.queryset.filter(**filter_dict).distinct() + def get_open_proposals(self): + """ + Returns the list of proposals anybody can access + :return: + """ + if os.environ.get("TEST_SECURITY_FLAG", False): + return ["OPEN", "private_dummy_project"] + else: + return ["OPEN", "lb27156"] + def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user if user.pk is None: From bfbcedd4d916587c28f03f718d4a235be8fc6e2a Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 12 Jan 2023 16:47:41 +0000 Subject: [PATCH 036/112] ISpyBSafeQuerySet now honours open-to-public Refactored to use Q-expressions for complex filtering --- api/security.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/api/security.py b/api/security.py index 6d6d790f..53a9327b 100644 --- a/api/security.py +++ b/api/security.py @@ -3,6 +3,7 @@ from wsgiref.util import FileWrapper from django.http import Http404 from django.http import HttpResponse +from django.db.models import Q from ispyb.connector.mysqlsp.main import ISPyBMySQLSPConnector as Connector from ispyb.connector.mysqlsp.main import ISPyBNoResultException from rest_framework import viewsets @@ -90,19 +91,22 @@ def get_queryset(self): proposal_list = self.get_proposals_for_user() # Add in the ones everyone has access to proposal_list.extend(self.get_open_proposals()) - # Must have a directly foreign key (project_id) for it to work - filter_dict = self.get_filter_dict(proposal_list) - return self.queryset.filter(**filter_dict).distinct() + # Must have a foreign key to a Project for this filter to work. + # get_q_filter() returns a Q expression for filtering + q_filter = self.get_q_filter(proposal_list) + return self.queryset.filter(q_filter).distinct() def get_open_proposals(self): """ - Returns the list of proposals anybody can access - :return: + Returns the list of proposals anybody can access. + This function is deprecated, instead we should move to the 'open_to_public' + field rather than using a built-in list of Projects. """ if os.environ.get("TEST_SECURITY_FLAG", False): - return ["OPEN", "private_dummy_project"] + return ["private_dummy_project"] else: - return ["OPEN", "lb27156"] + # A list of well-known (built-in) public Projects (Proposals/Visits) + return ["lb27156"] def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user @@ -178,14 +182,23 @@ def get_proposals_for_user(self): else: return self.get_proposals_for_user_from_django(user) - def get_filter_dict(self, proposal_list): + def get_q_filter(self, proposal_list): + """Returns a Q expression representing a (potentially complex) table filter. + """ if self.filter_permissions: - return {self.filter_permissions + "__title__in": proposal_list} + # Q-filter is based on the filter_permissions string + # whether the resultant Project title in the proposal list + # OR where the Project is 'open_to_public' + return Q(**{self.filter_permissions + "__title__in": proposal_list}) |\ + Q(**{self.filter_permissions + "__open_to_public": True}) else: # No filter permission? - # Assume this is QuerySet is used for the Project model. + # Assume this QuerySet is used for the Project model. # Added during 937 development (Access Control). - return {"title__in": proposal_list} + # + # Q-filter is based on the Project title being in the proposal list + # OR where the Project is 'open_to_public' + return Q(title__in=proposal_list) | Q(open_to_public=True) class ISpyBSafeStaticFiles: From 87b0219f2619c24b7ba379a3d323ebc35380f50a Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 12 Jan 2023 17:49:40 +0000 Subject: [PATCH 037/112] Attempt to fix unit tests --- api/security.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/security.py b/api/security.py index 53a9327b..89535f92 100644 --- a/api/security.py +++ b/api/security.py @@ -101,12 +101,13 @@ def get_open_proposals(self): Returns the list of proposals anybody can access. This function is deprecated, instead we should move to the 'open_to_public' field rather than using a built-in list of Projects. + We still add "OPEN" to the list for legacy testing. """ if os.environ.get("TEST_SECURITY_FLAG", False): - return ["private_dummy_project"] + return ["OPEN", "private_dummy_project"] else: # A list of well-known (built-in) public Projects (Proposals/Visits) - return ["lb27156"] + return ["OPEN", "lb27156"] def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user From d8898c81cf648cdb195321ea8323d99811422e86 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 16 Jan 2023 10:19:43 +0000 Subject: [PATCH 038/112] Adds debug logging to security --- api/security.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/security.py b/api/security.py index 89535f92..8bc4bcc3 100644 --- a/api/security.py +++ b/api/security.py @@ -1,5 +1,7 @@ +import logging import os import time + from wsgiref.util import FileWrapper from django.http import Http404 from django.http import HttpResponse @@ -11,6 +13,8 @@ from viewer.models import Project +logger = logging.getLogger(__name__) + USER_LIST_DICT = {} connector = os.environ.get('SECURITY_CONNECTOR', 'ispyb') @@ -91,6 +95,10 @@ def get_queryset(self): proposal_list = self.get_proposals_for_user() # Add in the ones everyone has access to proposal_list.extend(self.get_open_proposals()) + + logger.debug('is_authenticated=%s, proposal_list=%s', + self.request.user.is_authenticated, proposal_list) + # Must have a foreign key to a Project for this filter to work. # get_q_filter() returns a Q expression for filtering q_filter = self.get_q_filter(proposal_list) From 1a4c48aa194a459d25b7ce9a0234e8fba4a16002 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 17 Jan 2023 10:19:46 +0000 Subject: [PATCH 039/112] Protect against poor x/y dimensions --- api/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/utils.py b/api/utils.py index 4c358e7c..4c9ca680 100644 --- a/api/utils.py +++ b/api/utils.py @@ -156,8 +156,10 @@ def draw_mol(smiles, height=49, width=150, bondWidth=1, scaling=1.0, img_type=No # If you want to influence this use the scaling parameter. x, y = calc_bounds(conformer) dim_x, dim_y = calc_dims(x, y) - scale_x = width / dim_x - scale_y = height / dim_y + # Protect scaling from Div0. + # Scale factors are 0 if the corresponding dimension is not +ve, non=zero. + scale_x = width / dim_x if dim_x > 0 else 0 + scale_y = height / dim_y if dim_y > 0 else 0 scale = min(scale_x, scale_y) font = max(round(scale * scaling), 6) From 79abe965d0b1c1101837073951c9dcd74a5e840b Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 17 Jan 2023 10:53:41 +0000 Subject: [PATCH 040/112] Safer GetProp --- viewer/cset_upload.py | 21 ++++++++++++++++++--- viewer/tasks.py | 12 +++++++----- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/viewer/cset_upload.py b/viewer/cset_upload.py index 004f8e31..4fd79350 100644 --- a/viewer/cset_upload.py +++ b/viewer/cset_upload.py @@ -447,13 +447,28 @@ def task(self): def blank_mol_vals(sdf_file): + """Returns the submitter name, method and version (_Name) if present. + If not present the corresponding values are empty strings. + """ suppl = Chem.SDMolSupplier(sdf_file) + if not suppl: + return '', '', '' # print('%d mols detected (including blank mol)' % (len(suppl),)) blank_mol = suppl[0] + if not blank_mol: + return '', '', '' # Get submitter name/info for passing into upload to get unique name - submitter_name = blank_mol.GetProp('submitter_name') - submitter_method = blank_mol.GetProp('method') - version = blank_mol.GetProp('_Name') + submitter_name = '' + if blank_mol.HasProp('submitter_name'): + submitter_name = blank_mol.GetProp('submitter_name') + + submitter_method = '' + if blank_mol.HasProp('method'): + submitter_method = blank_mol.GetProp('method') + + version = '' + if blank_mol.HasProp('_Name'): + version = blank_mol.GetProp('_Name') return submitter_name, submitter_method, version diff --git a/viewer/tasks.py b/viewer/tasks.py index 96c05039..90017913 100644 --- a/viewer/tasks.py +++ b/viewer/tasks.py @@ -244,10 +244,6 @@ def validate_compound_set(task_params): # print('%d mols detected (including blank mol)' % (len(suppl),)) blank_mol = suppl[0] - # Get submitter name/info for passing into upload to get unique name - submitter_name = blank_mol.GetProp('submitter_name') - submitter_method = blank_mol.GetProp('method') - if blank_mol is None: validate_dict = add_warning(molecule_name='Blank Mol', field='N/A', @@ -260,6 +256,9 @@ def validate_compound_set(task_params): logger.warning('validate_compound_set() EXIT' ' user_id=%s sdf_file=%s validated=False', user_id, sdf_file) + # Can't get submitter name or method when there is now mol + submitter_name = '' + submitter_method = '' return (validate_dict, validated, sdf_file, target, zfile, submitter_name, submitter_method) @@ -295,7 +294,10 @@ def validate_compound_set(task_params): props = [key for key in list(mol.GetPropsAsDict().keys())] diff_list = np.setdiff1d(props, unique_props) for diff in diff_list: - add_warning(molecule_name=mol.GetProp('_Name'), + molecule_name = 'Unknown (no _Name property)' + if mol.HasProp('_Name'): + molecule_name = mol.GetProp('_Name') + add_warning(molecule_name=molecule_name, field='property (missing)', warning_string=f'{diff} property is missing from this molecule', validate_dict=validate_dict) From 3d7a9a539d9e52d1f66da8e98f996da610dcff6b Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 17 Jan 2023 11:31:54 +0000 Subject: [PATCH 041/112] More protection for GetProp calls --- viewer/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/viewer/tasks.py b/viewer/tasks.py index 90017913..1745e839 100644 --- a/viewer/tasks.py +++ b/viewer/tasks.py @@ -323,7 +323,10 @@ def validate_compound_set(task_params): for m in other_mols: if m: validate_dict = check_mol_props(m, validate_dict) - validate_dict = check_name_characters(m.GetProp('_Name'), validate_dict) + molecule_name = '' + if m.HasProp('_Name'): + molecule_name = m.GetProp('_Name') + validate_dict = check_name_characters(molecule_name, validate_dict) # validate_dict = check_pdb(m, validate_dict, target, zfile) validate_dict = check_refmol(m, validate_dict, target) validate_dict = check_field_populated(m, validate_dict) From b1dc8971d53796301260a3e5fa5b86a1d9d29a10 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 17 Jan 2023 14:38:20 +0000 Subject: [PATCH 042/112] Use of Squonk2 client 1.17 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd5d8e2e..1f595e05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.15.1 +im-squonk2-client==1.17.0 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 From 77f90111a41262f993a3c11ca2e71bd3e29e2d08 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 17 Jan 2023 14:48:58 +0000 Subject: [PATCH 043/112] New Squonk2 client --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1f595e05..15dac38e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.17.0 +im-squonk2-client==1.17.1 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 From cbd54dca815d4749584245343b7741ce3515dbe6 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 18 Jan 2023 09:56:15 +0000 Subject: [PATCH 044/112] Adds logging to security --- api/security.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/api/security.py b/api/security.py index 8bc4bcc3..e313ee86 100644 --- a/api/security.py +++ b/api/security.py @@ -59,6 +59,7 @@ def get_remote_conn(): # Assume the credentials are invalid if there is no host. # If a host is not defined other properties are useless. if not ispyb_credentials["host"]: + logger.debug("No ISPyB host - cannot return a connector") return None conn = SSHConnector(**ispyb_credentials) @@ -120,11 +121,14 @@ def get_open_proposals(self): def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user if user.pk is None: + logger.warning("user.pk is None") return [] else: - return list( + prop_ids = list( Project.objects.filter(user_id=user.pk).values_list("title", flat=True) ) + logger.debug("Got %s proposals: %s", len(prop_ids), prop_ids) + return prop_ids def needs_updating(self, user): global USER_LIST_DICT @@ -155,6 +159,9 @@ def get_proposals_for_user_from_ispyb(self, user): # First check if it's updated in the past 1 hour global USER_LIST_DICT + needs_updating = self.needs_updating(user) + logger.debug("needs_updating=%s", needs_updating) + if self.needs_updating(user): conn = None if connector == 'ispyb': @@ -162,9 +169,10 @@ def get_proposals_for_user_from_ispyb(self, user): if connector == 'ssh_ispyb': conn = get_remote_conn() - # If there is no connection (ISpyB credentials may be missing) + # If there is no connection (ISPyB credentials may be missing) # then there's nothing we can do except return an empty list. if conn is None: + logger.warning("Failed to get ISPyB connector") return [] rs = self.run_query_with_connector(conn=conn, user=user) @@ -175,18 +183,25 @@ def get_proposals_for_user_from_ispyb(self, user): prop_ids = list(set([str(x["proposalNumber"]) for x in rs])) prop_ids.extend(visit_ids) USER_LIST_DICT[user.username]["RESULTS"] = prop_ids + + logger.debug("Got %s proposals: %s", len(prop_ids), prop_ids) return prop_ids else: - return USER_LIST_DICT[user.username]["RESULTS"] + cached_prop_ids = USER_LIST_DICT[user.username]["RESULTS"] + logger.debug("Got %s cached proposals: %s", len(cached_prop_ids), cached_prop_ids) + return cached_prop_ids def get_proposals_for_user(self): user = self.request.user ispyb_user = os.environ.get("ISPYB_USER") + logger.debug("ispyb_user=%s", ispyb_user) if ispyb_user: + logger.debug("user.is_authenticated=%s", user.is_authenticated) if user.is_authenticated: return self.get_proposals_for_user_from_ispyb(user) else: + logger.debug("Got no proposals") return [] else: return self.get_proposals_for_user_from_django(user) From c87fdf6e55776b4ff949538121d878c2f3e148bb Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 18 Jan 2023 15:01:43 +0000 Subject: [PATCH 045/112] Fix security when needs-updating Also removes duplicates from returned proposals --- api/security.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/security.py b/api/security.py index e313ee86..2b25088f 100644 --- a/api/security.py +++ b/api/security.py @@ -95,7 +95,10 @@ def get_queryset(self): # The list of proposals this user can have proposal_list = self.get_proposals_for_user() # Add in the ones everyone has access to - proposal_list.extend(self.get_open_proposals()) + # (unless we already have it) + for open_proposal in self.get_open_proposals(): + if open_proposal not in proposal_list: + proposal_list.append(open_proposal) logger.debug('is_authenticated=%s, proposal_list=%s', self.request.user.is_authenticated, proposal_list) @@ -160,9 +163,9 @@ def get_proposals_for_user_from_ispyb(self, user): global USER_LIST_DICT needs_updating = self.needs_updating(user) - logger.debug("needs_updating=%s", needs_updating) + logger.debug("user=%s needs_updating=%s", user.username, needs_updating) - if self.needs_updating(user): + if needs_updating: conn = None if connector == 'ispyb': conn = get_conn() From 493f567aef06d529c746d75a8fd29ea5f0bcde7a Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 19 Jan 2023 12:34:45 +0000 Subject: [PATCH 046/112] mozilla_django_oidc logging now set to WARNING (was DEBUG) --- fragalysis/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragalysis/settings.py b/fragalysis/settings.py index 0d6b1129..364fe1d8 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -384,7 +384,7 @@ 'django': { 'level': 'WARNING'}, 'mozilla_django_oidc': { - 'level': 'DEBUG'}, + 'level': 'WARNING'}, 'urllib3': { 'level': 'WARNING'}}, 'root': { From 29565d9114c9933d8bd4a368784b38a7905006c7 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 19 Jan 2023 14:16:09 +0000 Subject: [PATCH 047/112] Better handling of SSHConnector() exceptions Logging should now contain a timezone --- api/security.py | 9 ++++++++- fragalysis/settings.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/security.py b/api/security.py index 2b25088f..2bac7254 100644 --- a/api/security.py +++ b/api/security.py @@ -62,7 +62,14 @@ def get_remote_conn(): logger.debug("No ISPyB host - cannot return a connector") return None - conn = SSHConnector(**ispyb_credentials) + # Try to get an SSH connection (aware that it might fail) + conn = None + try: + conn = SSHConnector(**ispyb_credentials) + except ValueError as cex: + logger.info("ssh_credentials=%s", ssh_credentials) + logger.error("Got ValueError exception getting SSH connection (%s)", cex) + return conn diff --git a/fragalysis/settings.py b/fragalysis/settings.py index 364fe1d8..727e7058 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -365,7 +365,7 @@ 'formatters': { 'simple': { 'format': '%(asctime)s %(name)s.%(funcName)s():%(lineno)s %(levelname)s # %(message)s', - 'datefmt': '%Y-%m-%dT%H:%M:%S'}}, + 'datefmt': '%Y-%m-%dT%H:%M:%S%z'}}, 'handlers': { 'console': { 'level': 'DEBUG', From 6adf240760e35be6daddad787f31e19d96d23edb Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 19 Jan 2023 14:52:45 +0000 Subject: [PATCH 048/112] paramiko log level now WARNING --- fragalysis/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fragalysis/settings.py b/fragalysis/settings.py index 727e7058..a69e5322 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -386,6 +386,8 @@ 'mozilla_django_oidc': { 'level': 'WARNING'}, 'urllib3': { + 'level': 'WARNING'}, + 'paramiko': { 'level': 'WARNING'}}, 'root': { 'level': LOGGING_FRAMEWORK_ROOT_LEVEL, From f43f17f7b511cb16dda0f309a72e7953f964bb2b Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 23 Jan 2023 14:07:34 +0000 Subject: [PATCH 049/112] Fixes loophole in Squonk2 - Now checks access ID is available to the user - Adds api.security and celery to logging config - Some squonk2 usage bug-fixes --- api/security.py | 10 ++-- fragalysis/settings.py | 4 ++ viewer/squonk2_agent.py | 66 +++++++++++++++++++++-- viewer/squonk_job_request.py | 41 +++++++------- viewer/views.py | 101 ++++++++++++++++++++++++++--------- 5 files changed, 171 insertions(+), 51 deletions(-) diff --git a/api/security.py b/api/security.py index 2bac7254..9aacfb3c 100644 --- a/api/security.py +++ b/api/security.py @@ -100,7 +100,7 @@ def get_queryset(self): Optionally restricts the returned purchases to a given proposals """ # The list of proposals this user can have - proposal_list = self.get_proposals_for_user() + proposal_list = self.get_proposals_for_user(self.request.user) # Add in the ones everyone has access to # (unless we already have it) for open_proposal in self.get_open_proposals(): @@ -201,9 +201,11 @@ def get_proposals_for_user_from_ispyb(self, user): logger.debug("Got %s cached proposals: %s", len(cached_prop_ids), cached_prop_ids) return cached_prop_ids - def get_proposals_for_user(self): - - user = self.request.user + def get_proposals_for_user(self, user): + """Returns a list of proposals (public and private) that the user has access to. + """ + assert user + ispyb_user = os.environ.get("ISPYB_USER") logger.debug("ispyb_user=%s", ispyb_user) if ispyb_user: diff --git a/fragalysis/settings.py b/fragalysis/settings.py index a69e5322..084acae8 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -379,8 +379,12 @@ 'filename': os.path.join(BASE_DIR, 'logs/backend.log'), 'formatter': 'simple'}}, 'loggers': { + 'api.security': { + 'level': 'WARNING'}, 'asyncio': { 'level': 'WARNING'}, + 'celery': { + 'level': 'WARNING'}, 'django': { 'level': 'WARNING'}, 'mozilla_django_oidc': { diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 84457b46..05c2c359 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -21,6 +21,7 @@ from requests import Response from wrapt import synchronized +from api.security import ISpyBSafeQuerySet from viewer.models import User, SessionProject, Project from viewer.models import Squonk2Project, Squonk2Org, Squonk2Unit @@ -67,7 +68,7 @@ _SQ2_PRODUCT_TYPE: str = 'DATA_MANAGER_PROJECT_TIER_SUBSCRIPTION' # True if the code's in Test Mode -_TEST_MODE: bool = True +_TEST_MODE: bool = False class Squonk2Agent: @@ -152,6 +153,12 @@ def __init__(self): self.__keycloak_hostname: str = '' self.__keycloak_realm: str = '' + # The Safe QuerySet from the security module. + # Used when we are given an access_id. + # It allows us to check that a user is permitted to use the access ID + # and relies on ISPyB credentials present in the environment. + self.__ispyb_safe_query_set: ISpyBSafeQuerySet = ISpyBSafeQuerySet() + def _get_user_name(self, user_id: int) -> str: # Gets the username (if id looks sensible) # or a fixed value (used for testing) @@ -162,7 +169,7 @@ def _get_user_name(self, user_id: int) -> str: user_name: str = self.__DUMMY_USER if user_id: - user: User = User.objects.filter(id=params.user_id).first() + user: User = User.objects.filter(id=user_id).first() assert user user_name = user.username assert user_name @@ -687,7 +694,7 @@ def ping(self) -> Squonk2AgentRv: return SuccessRv @synchronized - def run_job(self, params: RunJobParams) -> Squonk2AgentRv: + def can_run_job(self, params: RunJobParams) -> Squonk2AgentRv: """Executes a Job on a Squonk2 installation. """ assert params @@ -704,6 +711,49 @@ def run_job(self, params: RunJobParams) -> Squonk2AgentRv: _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) + # Ensure that the user is allowed to use the given access ID + user: User = User.objects.filter(id=params.user_id).first() + assert user + access_id: str = params.common.access_id + assert access_id + proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) + if not access_id in proposal_list: + msg = f'The user does not have access to {access_id}' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return SuccessRv + + @synchronized + def can_send(self, params: SendParams) -> Squonk2AgentRv: + """A blocking method that checks whether a user can send files to Squonk2. + """ + assert params + assert isinstance(params, SendParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API**MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Ensure that the user is allowed to use the access ID + user: User = User.objects.filter(id=params.user_id).first() + assert user + access_id: str = params.common.access_id + assert access_id + proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) + if not access_id in proposal_list: + msg = f'You cannot access {access_id}' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + return SuccessRv @synchronized @@ -726,6 +776,16 @@ def send(self, params: SendParams) -> Squonk2AgentRv: _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) + # Ensure that the user is allowed to use the access ID + user = None + access_id: str = params.common.access_id + assert access_id + proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) + if not access_id in proposal_list: + msg = f'The user does not have access to {access_id}' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + rv_u: Squonk2AgentRv = self._ensure_project(params.common) if not rv_u.success: msg = 'Failed to create corresponding Squonk2 Project' diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index e1f1cf0b..bcf5b090 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -18,7 +18,7 @@ JobRequest, JobFileTransfer ) from viewer.utils import create_squonk_job_request_url, get_https_host -from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent +from viewer.squonk2_agent import CommonParams, Squonk2AgentRv, Squonk2Agent, get_squonk2_agent logger = logging.getLogger(__name__) @@ -91,19 +91,18 @@ def create_squonk_job(request): auth_token = request.session['oidc_access_token'] logger.debug(auth_token) - squonk_job_name = request.data['squonk_job_name'] + access_id = request.data['access'] # The access ID/legacy Project record title target_id = request.data['target'] snapshot_id = request.data['snapshot'] - session_project_id = request.data['session_project'] - # The access ID (legacy Project record ID) - access_id = request.data['access'] + session_project_id = request.data['session_project'] + squonk_job_name = request.data['squonk_job_name'] squonk_job_spec = request.data['squonk_job_spec'] - logger.info('+ squonk_job_name=%s', squonk_job_name) + logger.info('+ access_id=%s', access_id) logger.info('+ target_id=%s', target_id) logger.info('+ snapshot_id=%s', snapshot_id) logger.info('+ session_project_id=%s', session_project_id) - logger.info('+ access_id=%s', access_id) + logger.info('+ squonk_job_name=%s', squonk_job_name) logger.info('+ squonk_job_spec=%s', squonk_job_spec) job_transfers = JobFileTransfer.objects.filter(snapshot=snapshot_id) @@ -119,25 +118,31 @@ def create_squonk_job(request): job_transfer.transfer_status) raise ValueError('Job Transfer not complete') + logger.info('+ Calling ensure_project() to get the Squonk2 Project...') + # This requires a Squonk2 Project (created by the Squonk2Agent). # It may be an existing project, or it might be a new project. + user = request.user common_params = CommonParams(user_id=user.id, access_id=access_id, target_id=target_id, session_id=session_project_id) sq2_rv = _SQ2A.ensure_project(common_params) if not sq2_rv.success: - msg = f'JobTransfer failed to get a Squonk2 Project' \ - f' for User {user.username}, Access ID {access_id},' \ - f' Target ID {target_id}, and SessionProject ID {session_id}.' \ - f' Returning the message "{ep_rv.msg}".' \ - ' Cannot continue' - content = {'message': msg} + msg = f'JobTransfer failed to get/create a Squonk2 Project' \ + f' for User "{user.username}", Access ID {access_id},' \ + f' Target ID {target_id}, and SessionProject ID {session_project_id}.' \ + f' Got "{sq2_rv.msg}".' \ + ' Cannot continue' logger.error(msg) - return Response(content, - status=status.HTTP_404_NOT_FOUND) - # The project UUID is in the response msg (a Squonk2Project object) - squonk2_project = sq2_rv.msg.uuid + raise ValueError(msg) + + # The Squonk2Project record is in the response msg + squonk2_project_uuid = sq2_rv.msg.uuid + squonk2_unit_name = sq2_rv.msg.unit.name + squonk2_unit_uuid = sq2_rv.msg.unit.uuid + logger.info('+ ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)', + squonk2_project_uuid, squonk2_unit_name, squonk2_unit_uuid) job_request = JobRequest() job_request.squonk_job_name = squonk_job_name @@ -147,7 +152,7 @@ def create_squonk_job(request): # We should use a foreign key, # but to avoid migration issues with the existing code # we continue to use the project UUID string field. - job_request.squonk_project = squonk2_project + job_request.squonk_project = squonk2_project_uuid job_request.squonk_job_spec = squonk_job_spec # Saving creates the uuid for the callback diff --git a/viewer/views.py b/viewer/views.py index 03fe0460..f96ba0ec 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -63,7 +63,7 @@ ) from viewer import filters from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent -from viewer.squonk2_agent import CommonParams +from viewer.squonk2_agent import CommonParams, SendParams, RunJobParams from .forms import CSetForm, CSetUpdateForm, TSetForm from .tasks import ( @@ -3126,18 +3126,41 @@ def create(self, request): content = {'Only authenticated users can transfer files'} return Response(content, status=status.HTTP_403_FORBIDDEN) - # Can't use this method if the Squonk2 agent is not - if not _SQ2A.configured(): - content = {'The Squonk2 Agent is not configured'} + # Can't use this method if the Squonk2 agent is not configured + sq2a_rv = _SQ2A.configured() + if not sq2a_rv.success: + content = {f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'} return Response(content, status=status.HTTP_403_FORBIDDEN) + # Collect expected API parameters.... + access_id = request.data['access'] # The access ID/legacy Project record title target_id = request.data['target'] - target = Target.objects.get(id=target_id) snapshot_id = request.data['snapshot'] session_project_id = request.data['session_project'] - # The access ID (legacy Project record ID) - access_id = request.data['access'] + logger.info('+ user="%s" (id=%s)', user.username, user.id) + logger.info('+ access_id=%s', access_id) + logger.info('+ target_id=%s', target_id) + logger.info('+ snapshot_id=%s', snapshot_id) + logger.info('+ session_project_id=%s', session_project_id) + + target = Target.objects.get(id=target_id) + assert target + + # Check the user can use this Squonk2 facility. + # To do this we need to setup a couple of API parameter objects. + sq2a_common_params: CommonParams = CommonParams(user_id=user.id, + access_id=access_id, + session_id=session_project_id, + target_id=target_id) + sq2a_send_params: SendParams = SendParams(common=sq2a_common_params, + snapshot_id=snapshot_id) + sq2a_rv: Squonk2AgentRv = _SQ2A.can_send(sq2a_send_params) + if not sq2a_rv.success: + content = {f'You cannot do this ({sq2a_rv.msg})'} + return Response(content, status=status.HTTP_403_FORBIDDEN) + + # Check the presense of the files expected to be transferred error, proteins, compounds = check_file_transfer(request) if error: return Response(error['message'], status=error['status']) @@ -3161,11 +3184,6 @@ def create(self, request): # This is "understood" by the celery task (which uses this constant). # e.g. 'fragalysis-files' transfer_root = settings.SQUONK2_MEDIA_DIRECTORY - - logger.info('+ target_id=%s', target_id) - logger.info('+ snapshot_id=%s', snapshot_id) - logger.info('+ session_project_id=%s', session_project_id) - logger.info('+ access_id=%s', access_id) logger.info('+ transfer_root=%s', transfer_root) if job_transfer: @@ -3200,6 +3218,7 @@ def create(self, request): else: # Create new file transfer job + logger.info('+ Calling ensure_project() to get the Squonk2 Project...') # This requires a Squonk2 Project (created by the Squonk2Agent). # It may be an existing project, or it might be a new project. @@ -3209,17 +3228,21 @@ def create(self, request): session_id=session_project_id) sq2_rv = _SQ2A.ensure_project(common_params) if not sq2_rv.success: - msg = f'JobTransfer failed to get a Squonk2 Project' \ - f' for User {user.username}, Access ID {access_id},' \ - f' Target ID {target_id}, and SessionProject ID {session_id}.' \ - f' Returning the message "{sq2_rv.msg}".' \ + msg = f'Failed to get/create a Squonk2 Project' \ + f' for User "{user.username}", Access ID {access_id},' \ + f' Target ID {target_id}, and SessionProject ID {session_project_id}.' \ + f' Got "{sq2_rv.msg}".' \ ' Cannot continue' content = {'message': msg} logger.error(msg) - return Response(content, - status=status.HTTP_404_NOT_FOUND) - # The project UUID is in the response msg (a Squonk2Project object) - squonk2_project = sq2_rv.msg.uuid + return Response(content, status=status.HTTP_404_NOT_FOUND) + + # The Squonk2Project record is in the response msg + squonk2_project_uuid = sq2_rv.msg.uuid + squonk2_unit_name = sq2_rv.msg.unit.name + squonk2_unit_uuid = sq2_rv.msg.unit.uuid + logger.info('+ ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)', + squonk2_project_uuid, squonk2_unit_name, squonk2_unit_uuid) job_transfer = JobFileTransfer() job_transfer.user = request.user @@ -3228,7 +3251,7 @@ def create(self, request): # We should use a foreign key, # but to avoid migration issues with the existing code # we continue to use the project UUID string field. - job_transfer.squonk_project = squonk2_project + job_transfer.squonk_project = squonk2_project_uuid job_transfer.target = Target.objects.get(id=target_id) job_transfer.snapshot = Snapshot.objects.get(id=snapshot_id) @@ -3395,11 +3418,37 @@ def create(self, request): content = {'Only authenticated users can run jobs'} return Response(content, status=status.HTTP_403_FORBIDDEN) - # Can't use this method if the squonk variables are not set! - # Do this for JobRequest and FileTransfer - sq2_rv = _SQ2A.configured - if not sq2_rv.success: - content = {f'The Squonk2 Agent is not configured ({sqa_rv.msg})'} + # Can't use this method if the Squonk2 agent is not configured + sq2a_rv = _SQ2A.configured() + if not sq2a_rv.success: + content = {f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'} + return Response(content, status=status.HTTP_403_FORBIDDEN) + + # Collect expected API parameters.... + target_id = request.data['target'] + snapshot_id = request.data['snapshot'] + session_project_id = request.data['session_project'] + access_id = request.data['access'] # The access ID/legacy Project record title + + logger.info('+ user="%s" (id=%s)', user.username, user.id) + logger.info('+ access_id=%s', access_id) + logger.info('+ target_id=%s', target_id) + logger.info('+ snapshot_id=%s', snapshot_id) + logger.info('+ session_project_id=%s', session_project_id) + + # Check the user can use this Squonk2 facility. + # To do this we need to setup a couple of API parameter objects. + # We don't (at this point) care about the Job spec or callback URL. + sq2a_common_params: CommonParams = CommonParams(user_id=user.id, + access_id=access_id, + session_id=session_project_id, + target_id=target_id) + sq2a_run_job_params: RunJobParams = RunJobParams(common=sq2a_common_params, + job_spec=None, + callback_url=None) + sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params) + if not sq2a_rv.success: + content = {f'You cannot do this ({sq2a_rv.msg})'} return Response(content, status=status.HTTP_403_FORBIDDEN) try: From ef1defd5a54a2297d635889ff27a2da51ebddba3 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 24 Jan 2023 12:44:55 +0000 Subject: [PATCH 050/112] Logs ISPyB response Doc tweaks --- api/security.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/api/security.py b/api/security.py index 9aacfb3c..af59def0 100644 --- a/api/security.py +++ b/api/security.py @@ -181,22 +181,31 @@ def get_proposals_for_user_from_ispyb(self, user): # If there is no connection (ISPyB credentials may be missing) # then there's nothing we can do except return an empty list. + # Otherwise run a query for the user. if conn is None: logger.warning("Failed to get ISPyB connector") return [] - rs = self.run_query_with_connector(conn=conn, user=user) - + logger.debug("Connector query rs=%s", rs) + + # Iterate through the response and return the 'proposalNumber' (proposals) + # and one with the 'proposalNumber' and 'sessionNumber' (visits), + # e.g. ["12345", "12345-1"]. + # + # These strings would normally correspond to a title value + # in a Project record. visit_ids = list(set([ str(x["proposalNumber"]) + "-" + str(x["sessionNumber"]) for x in rs ])) prop_ids = list(set([str(x["proposalNumber"]) for x in rs])) prop_ids.extend(visit_ids) - USER_LIST_DICT[user.username]["RESULTS"] = prop_ids - logger.debug("Got %s proposals: %s", len(prop_ids), prop_ids) + + # Cache the result and return the result for the user + USER_LIST_DICT[user.username]["RESULTS"] = prop_ids return prop_ids else: + # Return the previous query (cached) cached_prop_ids = USER_LIST_DICT[user.username]["RESULTS"] logger.debug("Got %s cached proposals: %s", len(cached_prop_ids), cached_prop_ids) return cached_prop_ids From abf8e2795721524fa902f31665e5b851e578cff5 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 24 Jan 2023 14:33:25 +0000 Subject: [PATCH 051/112] Adds Project reference to SessionProject record --- .../migrations/0027_sessionproject_project.py | 19 +++++++++++++++++++ viewer/models.py | 9 ++++++--- viewer/serializers.py | 19 ++++++++++--------- 3 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 viewer/migrations/0027_sessionproject_project.py diff --git a/viewer/migrations/0027_sessionproject_project.py b/viewer/migrations/0027_sessionproject_project.py new file mode 100644 index 00000000..56bfb07e --- /dev/null +++ b/viewer/migrations/0027_sessionproject_project.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.14 on 2023-01-24 14:23 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0026_add_project_open_to_public_field'), + ] + + operations = [ + migrations.AddField( + model_name='sessionproject', + name='project', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='viewer.project'), + ), + ] diff --git a/viewer/models.py b/viewer/models.py index 7a29d4c9..dc2c3a0a 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -404,7 +404,8 @@ class Meta: # Start of Session Project class SessionProject(models.Model): - """Django model for holding information about a fragalysis user project - a set of sessions saved by a user + """Django model for holding information about a fragalysis user Session Project + - a set of sessions saved by a user that belong to a Target and Project. Parameters ---------- @@ -415,17 +416,19 @@ class SessionProject(models.Model): description: Charfield A short user-defined description for the project target: ForeignKey - Foreign Key link to the relevent project target + Foreign Key link to the relevant project target + project: ForeignKey + Foreign Key link to the relevant project (optional for legacy reasons) author: ForeignKey A link to the user that created the project tags: TextField A comma separated list of user-defined tags - for searching and tagging projects - """ title = models.CharField(max_length=200) init_date = models.DateTimeField(default=timezone.now) description = models.CharField(max_length=255, default='') target = models.ForeignKey(Target, on_delete=models.CASCADE) + project = models.ForeignKey(Project, null=True, on_delete=models.CASCADE) author = models.ForeignKey(User, null=True, on_delete=models.CASCADE) tags = models.TextField(default='[]') diff --git a/viewer/serializers.py b/viewer/serializers.py index 96042098..d77f3480 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -287,16 +287,12 @@ class ProjectSerializer(serializers.ModelSerializer): # Field name translation (prior to refactoring the Model) # 'tas' is the new name for 'title' - target_access_string = serializers.SerializerMethodField('get_tas') + target_access_string = serializers.SerializerMethodField() # 'authority' is the (as yet to be implemented) origin of the TAS # For now this is fixed at "DIAMOND-ISPYB" - authority = serializers.SerializerMethodField('get_authority') + authority = serializers.SerializerMethodField() - class Meta: - model = Project - fields = ("id", "target_access_string", "init_date", "authority", "open_to_public") - - def get_tas(self, instance): + def get_target_access_string(self, instance): return instance.title def get_authority(self, instance): @@ -305,6 +301,10 @@ def get_authority(self, instance): del instance return "DIAMOND-ISPYB" + class Meta: + model = Project + fields = ("id", "target_access_string", "init_date", "authority", "open_to_public") + class MolImageSerializer(serializers.ModelSerializer): @@ -433,7 +433,8 @@ class Meta: # GET class SessionProjectReadSerializer(serializers.ModelSerializer): target = TargetSerializer(read_only=True) - author = UserSerializer(read_only=True) + project = ProjectSerializer(read_only=True) + target = TargetSerializer(read_only=True) # This is for the new tags functionality. The old tags field is left here for backwards # compatibility session_project_tags = serializers.SerializerMethodField() @@ -445,7 +446,7 @@ def get_session_project_tags(self, obj): class Meta: model = SessionProject fields = ('id', 'title', 'init_date', 'description', - 'target', 'author', 'tags', 'session_project_tags') + 'target', 'project', 'author', 'tags', 'session_project_tags') # (POST, PUT, PATCH) From d7314e69351d91871a5f4b4739e7bfe8abeb09b9 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 24 Jan 2023 15:31:27 +0000 Subject: [PATCH 052/112] Corrected security (to support 'proposalCode') --- api/security.py | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/api/security.py b/api/security.py index af59def0..50b01257 100644 --- a/api/security.py +++ b/api/security.py @@ -188,22 +188,49 @@ def get_proposals_for_user_from_ispyb(self, user): rs = self.run_query_with_connector(conn=conn, user=user) logger.debug("Connector query rs=%s", rs) + # Typically you'll find the following fields in each item + # in the rs response; - + # + # 'id': 0000000, + # 'proposalId': 00000, + # 'startDate': datetime.datetime(2022, 12, 1, 15, 56, 30) + # 'endDate': datetime.datetime(2022, 12, 3, 18, 34, 9) + # 'beamline': 'i00-0' + # 'proposalCode': 'lb' + # 'proposalNumber': '12345' + # 'sessionNumber': 1 + # 'comments': None + # 'personRoleOnSession': 'Data Access' + # 'personRemoteOnSession': 1 + # # Iterate through the response and return the 'proposalNumber' (proposals) # and one with the 'proposalNumber' and 'sessionNumber' (visits), # e.g. ["12345", "12345-1"]. # # These strings would normally correspond to a title value # in a Project record. - visit_ids = list(set([ - str(x["proposalNumber"]) + "-" + str(x["sessionNumber"]) for x in rs - ])) - prop_ids = list(set([str(x["proposalNumber"]) for x in rs])) - prop_ids.extend(visit_ids) - logger.debug("Got %s proposals: %s", len(prop_ids), prop_ids) + # + # To maintain backward compatibility we create a list of + # raw proposals and visits and a duplicate set with the 'proposalCode' + # prefix (converted to upper-case). The proposalCode is 'aa' but 'AA' + # will be in the project record. + # + # e.g. we eventually get this sort of list: - + # + # ["12345", "12345-1", "LB12345", "LB12345-1"] + prop_id_set = set() + for record in rs: + proposal_str = f'{record["proposalNumber"]}' + visit_str = f'{proposal_str}-{record["sessionNumber"]}' + prop_id_set.update([proposal_str, visit_str]) + if record["proposalCode"]: + proposalCode = str(record["proposalCode"]).upper() + prop_id_set.update([f'{proposalCode}{proposal_str}', f'{proposalCode}{visit_str}']) + logger.debug("Got %s proposals: %s", len(prop_id_set), prop_id_set) # Cache the result and return the result for the user - USER_LIST_DICT[user.username]["RESULTS"] = prop_ids - return prop_ids + USER_LIST_DICT[user.username]["RESULTS"] = list(prop_id_set) + return USER_LIST_DICT[user.username]["RESULTS"] else: # Return the previous query (cached) cached_prop_ids = USER_LIST_DICT[user.username]["RESULTS"] From cab571d1f7c064d4a97d2dbfed83526df1e33764 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 25 Jan 2023 14:34:50 +0000 Subject: [PATCH 053/112] Proposals now prefix with code (un-treated) i.e. "lb12345-12" not "LB12345-12" --- api/security.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/api/security.py b/api/security.py index 50b01257..1b032b3e 100644 --- a/api/security.py +++ b/api/security.py @@ -204,28 +204,28 @@ def get_proposals_for_user_from_ispyb(self, user): # 'personRemoteOnSession': 1 # # Iterate through the response and return the 'proposalNumber' (proposals) - # and one with the 'proposalNumber' and 'sessionNumber' (visits), - # e.g. ["12345", "12345-1"]. + # and one with the 'proposalNumber' and 'sessionNumber' (visits), each + # prefixed by the `proposalCode` (if present). # - # These strings would normally correspond to a title value - # in a Project record. + # These strings should correspond to a title value in a Project record. # - # To maintain backward compatibility we create a list of - # raw proposals and visits and a duplicate set with the 'proposalCode' - # prefix (converted to upper-case). The proposalCode is 'aa' but 'AA' - # will be in the project record. - # - # e.g. we eventually get this sort of list: - + # We should get this sort of list: - # - # ["12345", "12345-1", "LB12345", "LB12345-1"] + # ["lb12345", "lb12345-1"] + # -- - + # | ----- | + # proposalCode | | + # proposalCode prop_id_set = set() for record in rs: - proposal_str = f'{record["proposalNumber"]}' - visit_str = f'{proposal_str}-{record["sessionNumber"]}' - prop_id_set.update([proposal_str, visit_str]) - if record["proposalCode"]: - proposalCode = str(record["proposalCode"]).upper() - prop_id_set.update([f'{proposalCode}{proposal_str}', f'{proposalCode}{visit_str}']) + pc_str = "" + if "proposalCode" in record and record["proposalCode"]: + pc_str = f'{record["proposalCode"]}' + pn_str = f'{record["proposalNumber"]}' + sn_str = f'{record["sessionNumber"]}' + proposal_str = f'{pc_str}{pn_str}' + proposal_visit_str = f'{proposal_str}-{sn_str}' + prop_id_set.update([proposal_str, proposal_visit_str]) logger.debug("Got %s proposals: %s", len(prop_id_set), prop_id_set) # Cache the result and return the result for the user From ab90c4c5f6ff20ef82ccdf27119584bf2054c0f0 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 25 Jan 2023 14:52:32 +0000 Subject: [PATCH 054/112] Security log now INFO Now always shows collected proposals (when out of date) --- api/security.py | 12 ++++++++++-- fragalysis/settings.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/api/security.py b/api/security.py index 1b032b3e..466de6cb 100644 --- a/api/security.py +++ b/api/security.py @@ -141,6 +141,10 @@ def get_proposals_for_user_from_django(self, user): return prop_ids def needs_updating(self, user): + """Returns true of the data collected for a user is out of date. + It's simple, we just record the last collected timestamp and consider it + 'out of date' (i.e. more than an hour old). + """ global USER_LIST_DICT update_window = 3600 @@ -226,13 +230,17 @@ def get_proposals_for_user_from_ispyb(self, user): proposal_str = f'{pc_str}{pn_str}' proposal_visit_str = f'{proposal_str}-{sn_str}' prop_id_set.update([proposal_str, proposal_visit_str]) - logger.debug("Got %s proposals: %s", len(prop_id_set), prop_id_set) + + # Always display the collected results for the user. + # These will be cached. + logger.info("Got %s proposals (%s): %s", + len(prop_id_set), user.username, prop_id_set) # Cache the result and return the result for the user USER_LIST_DICT[user.username]["RESULTS"] = list(prop_id_set) return USER_LIST_DICT[user.username]["RESULTS"] else: - # Return the previous query (cached) + # Return the previous query (cached for an hour) cached_prop_ids = USER_LIST_DICT[user.username]["RESULTS"] logger.debug("Got %s cached proposals: %s", len(cached_prop_ids), cached_prop_ids) return cached_prop_ids diff --git a/fragalysis/settings.py b/fragalysis/settings.py index 084acae8..c6652f99 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -380,7 +380,7 @@ 'formatter': 'simple'}}, 'loggers': { 'api.security': { - 'level': 'WARNING'}, + 'level': 'INFO'}, 'asyncio': { 'level': 'WARNING'}, 'celery': { From 98a7168eb4af8526931e80a0363de2830cdf4d8b Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 25 Jan 2023 15:10:38 +0000 Subject: [PATCH 055/112] Doc tweak --- api/security.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/security.py b/api/security.py index 466de6cb..a9fdd694 100644 --- a/api/security.py +++ b/api/security.py @@ -211,15 +211,17 @@ def get_proposals_for_user_from_ispyb(self, user): # and one with the 'proposalNumber' and 'sessionNumber' (visits), each # prefixed by the `proposalCode` (if present). # - # These strings should correspond to a title value in a Project record. + # Codes are expected to consist of 2 letters. + # Typically: lb, mx, nt, nr, bi # - # We should get this sort of list: - + # These strings should correspond to a title value in a Project record. + # and should get this sort of list: - # # ["lb12345", "lb12345-1"] # -- - # | ----- | - # proposalCode | | - # proposalCode + # Code | Session + # Proposal prop_id_set = set() for record in rs: pc_str = "" From a19370b5639395c6cf52c726ad55a922df63b0ee Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 2 Feb 2023 11:42:05 +0000 Subject: [PATCH 056/112] Fix user_id reference --- viewer/squonk2_agent.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 05c2c359..9898adc5 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -496,7 +496,7 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: return Squonk2AgentRv(success=True, msg=sq2_unit) - def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: + def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: """Gets or creates a Squonk2 Project, used as the destination of files and job executions. Each project requires an AS Product (tied to the User and Session) and Unit (tied to the Proposal/Project). @@ -509,17 +509,17 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: For testing the target and user IDs are permitted to be 0. """ - assert params - assert isinstance(params, CommonParams) + assert c_params + assert isinstance(c_params, CommonParams) # A Squonk2Unit must exist for the Target Access String. - rv: Squonk2AgentRv = self._ensure_unit(params.access_id) + rv: Squonk2AgentRv = self._ensure_unit(c_params.access_id) if not rv.success: return rv unit: Squonk2Unit = rv.msg - user_name: str = self._get_user_name(params.user_id) - session_title: str = self._get_session_title(params.session_id) + user_name: str = self._get_user_name(c_params.user_id) + session_title: str = self._get_session_title(c_params.session_id) assert user_name assert session_title @@ -530,7 +530,7 @@ def _ensure_project(self, params: CommonParams) -> Squonk2AgentRv: _LOGGER.info(msg) # Need to call upon Squonk2 to create a 'Product' # (and corresponding 'Product'). - rv = self._create_product_and_project(unit, user_name, session_title, params) + rv = self._create_product_and_project(unit, user_name, session_title, c_params) if not rv.success: msg = f'Failed creating AS Product or DM Project ({rv.msg})' _LOGGER.error(msg) @@ -694,11 +694,11 @@ def ping(self) -> Squonk2AgentRv: return SuccessRv @synchronized - def can_run_job(self, params: RunJobParams) -> Squonk2AgentRv: + def can_run_job(self, rj_params: RunJobParams) -> Squonk2AgentRv: """Executes a Job on a Squonk2 installation. """ - assert params - assert isinstance(params, RunJobParams) + assert rj_params + assert isinstance(rj_params, RunJobParams) if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' @@ -712,9 +712,9 @@ def can_run_job(self, params: RunJobParams) -> Squonk2AgentRv: return Squonk2AgentRv(success=False, msg=msg) # Ensure that the user is allowed to use the given access ID - user: User = User.objects.filter(id=params.user_id).first() + user: User = User.objects.filter(id=rj_params.common.user_id).first() assert user - access_id: str = params.common.access_id + access_id: str = rj_params.common.access_id assert access_id proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) if not access_id in proposal_list: @@ -725,11 +725,11 @@ def can_run_job(self, params: RunJobParams) -> Squonk2AgentRv: return SuccessRv @synchronized - def can_send(self, params: SendParams) -> Squonk2AgentRv: + def can_send(self, s_params: SendParams) -> Squonk2AgentRv: """A blocking method that checks whether a user can send files to Squonk2. """ - assert params - assert isinstance(params, SendParams) + assert s_params + assert isinstance(s_params, SendParams) if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' @@ -744,9 +744,9 @@ def can_send(self, params: SendParams) -> Squonk2AgentRv: return Squonk2AgentRv(success=False, msg=msg) # Ensure that the user is allowed to use the access ID - user: User = User.objects.filter(id=params.user_id).first() + user: User = User.objects.filter(id=s_params.common.user_id).first() assert user - access_id: str = params.common.access_id + access_id: str = s_params.common.access_id assert access_id proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) if not access_id in proposal_list: From 9ef222c31a088e354921c969410cfa1319fe4931 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 2 Feb 2023 12:51:50 +0000 Subject: [PATCH 057/112] Attempt to fix TAS verification (was using ID) --- viewer/squonk2_agent.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 9898adc5..4710ecc1 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -154,7 +154,7 @@ def __init__(self): self.__keycloak_realm: str = '' # The Safe QuerySet from the security module. - # Used when we are given an access_id. + # Used when we are given a tas (target access string). # It allows us to check that a user is permitted to use the access ID # and relies on ISPyB credentials present in the environment. self.__ispyb_safe_query_set: ISpyBSafeQuerySet = ISpyBSafeQuerySet() @@ -716,9 +716,12 @@ def can_run_job(self, rj_params: RunJobParams) -> Squonk2AgentRv: assert user access_id: str = rj_params.common.access_id assert access_id + target_access_string = self._get_target_access_string(access_id) + assert target_access_string proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) - if not access_id in proposal_list: - msg = f'The user does not have access to {access_id}' + if not target_access_string in proposal_list: + msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ + f' (access_id={access_id}). Only {proposal_list})' _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -748,9 +751,12 @@ def can_send(self, s_params: SendParams) -> Squonk2AgentRv: assert user access_id: str = s_params.common.access_id assert access_id + target_access_string = self._get_target_access_string(access_id) + assert target_access_string proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) - if not access_id in proposal_list: - msg = f'You cannot access {access_id}' + if not target_access_string in proposal_list: + msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ + f' (access_id={access_id}). Only {proposal_list})' _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -780,9 +786,12 @@ def send(self, params: SendParams) -> Squonk2AgentRv: user = None access_id: str = params.common.access_id assert access_id + target_access_string = self._get_target_access_string(access_id) + assert target_access_string proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) - if not access_id in proposal_list: - msg = f'The user does not have access to {access_id}' + if not target_access_string in proposal_list: + msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ + f'(access_id={access_id}). Only {proposal_list})' _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) From 68d7b6d06f16dd7a5a5d68ab470a7d7f90391e6b Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 2 Feb 2023 14:12:18 +0000 Subject: [PATCH 058/112] Adds lb00000 to set of open/built-in projects Fix for typo in Squonk2Agent --- api/security.py | 4 ++-- viewer/squonk2_agent.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/security.py b/api/security.py index a9fdd694..bd2c73e3 100644 --- a/api/security.py +++ b/api/security.py @@ -123,10 +123,10 @@ def get_open_proposals(self): We still add "OPEN" to the list for legacy testing. """ if os.environ.get("TEST_SECURITY_FLAG", False): - return ["OPEN", "private_dummy_project"] + return ["lb00000", "OPEN", "private_dummy_project"] else: # A list of well-known (built-in) public Projects (Proposals/Visits) - return ["OPEN", "lb27156"] + return ["lb00000", "OPEN", "lb27156"] def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 4710ecc1..cb8aa8bf 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -8,7 +8,7 @@ from collections import namedtuple import logging import os -from typing import List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from urllib.parse import ParseResult, urlparse from urllib3.exceptions import InsecureRequestWarning from urllib3 import disable_warnings @@ -202,7 +202,7 @@ def _get_session_title(self, session_id: int) -> str: session_title: str = self.__DUMMY_SESSION_TITLE if session_id: - session_project: SessionProject = SessionProject.objects.filter(id=params.session_id).first() + session_project: SessionProject = SessionProject.objects.filter(id=session_id).first() assert session_project session_title = session_project.title assert session_title @@ -371,7 +371,7 @@ def _delete_dm_project(self, project_uuid: str) -> None: dm_rv: DmApiRv = DmApi.delete_project(self.__org_owner_dm_token, project_id=project_uuid) - if not as_rv.success: + if not dm_rv.success: _LOGGER.error('Failed to delete DM Project %s', project_uuid) return From c25094ca668cff28225eb33a7df979bcf25bbe12 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 2 Feb 2023 14:34:31 +0000 Subject: [PATCH 059/112] Fix access checks --- viewer/squonk2_agent.py | 98 ++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index cb8aa8bf..1078063d 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -465,9 +465,6 @@ def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: """ assert self.__org_record - target_access_string = self._get_target_access_string(access_id) - assert target_access_string - # Now we check and create a Squonk2Unit... unit_name_truncated, unit_name_full = self._build_unit_name(target_access_string) sq2_unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(name=unit_name_full).first() @@ -512,8 +509,13 @@ def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: assert c_params assert isinstance(c_params, CommonParams) + + target_access_string = self._get_target_access_string(access_id) + assert target_access_string + + # A Squonk2Unit must exist for the Target Access String. - rv: Squonk2AgentRv = self._ensure_unit(c_params.access_id) + rv: Squonk2AgentRv = self._ensure_unit(target_access_string) if not rv.success: return rv unit: Squonk2Unit = rv.msg @@ -552,6 +554,25 @@ def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: return Squonk2AgentRv(success=True, msg=sq2_project) + def _verify_access(self, c_params: CommonParams) -> Squonk2AgentRv: + """Checks the user has access to the project. + """ + # Ensure that the user is allowed to use the given access ID + user: User = User.objects.filter(id=c_params.user_id).first() + assert user + access_id: str = c_params.access_id + assert access_id + target_access_string = self._get_target_access_string(access_id) + assert target_access_string + proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) + if not target_access_string in proposal_list: + msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ + f' (access_id={access_id}). Only {proposal_list})' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return SuccessRv + @synchronized def get_ui_url(self): """Returns the UI URL, if configured. @@ -711,21 +732,7 @@ def can_run_job(self, rj_params: RunJobParams) -> Squonk2AgentRv: _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - # Ensure that the user is allowed to use the given access ID - user: User = User.objects.filter(id=rj_params.common.user_id).first() - assert user - access_id: str = rj_params.common.access_id - assert access_id - target_access_string = self._get_target_access_string(access_id) - assert target_access_string - proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) - if not target_access_string in proposal_list: - msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ - f' (access_id={access_id}). Only {proposal_list})' - _LOGGER.warning(msg) - return Squonk2AgentRv(success=False, msg=msg) - - return SuccessRv + return self._verify_access(c_params=rj_params.common) @synchronized def can_send(self, s_params: SendParams) -> Squonk2AgentRv: @@ -746,29 +753,15 @@ def can_send(self, s_params: SendParams) -> Squonk2AgentRv: _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - # Ensure that the user is allowed to use the access ID - user: User = User.objects.filter(id=s_params.common.user_id).first() - assert user - access_id: str = s_params.common.access_id - assert access_id - target_access_string = self._get_target_access_string(access_id) - assert target_access_string - proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) - if not target_access_string in proposal_list: - msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ - f' (access_id={access_id}). Only {proposal_list})' - _LOGGER.warning(msg) - return Squonk2AgentRv(success=False, msg=msg) - - return SuccessRv + return self._verify_access(c_params=s_params.common) @synchronized - def send(self, params: SendParams) -> Squonk2AgentRv: + def send(self, s_params: SendParams) -> Squonk2AgentRv: """A blocking method that takes care of sending a set of files to the configured Squonk2 installation. """ - assert params - assert isinstance(params, SendParams) + assert s_params + assert isinstance(s_params, SendParams) if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' @@ -782,20 +775,11 @@ def send(self, params: SendParams) -> Squonk2AgentRv: _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - # Ensure that the user is allowed to use the access ID - user = None - access_id: str = params.common.access_id - assert access_id - target_access_string = self._get_target_access_string(access_id) - assert target_access_string - proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) - if not target_access_string in proposal_list: - msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ - f'(access_id={access_id}). Only {proposal_list})' - _LOGGER.warning(msg) - return Squonk2AgentRv(success=False, msg=msg) + rv_access: Squonk2AgentRv = self._verify_access(s_params.common) + if not rv_access.success: + return rv_access - rv_u: Squonk2AgentRv = self._ensure_project(params.common) + rv_u: Squonk2AgentRv = self._ensure_project(s_params.common) if not rv_u.success: msg = 'Failed to create corresponding Squonk2 Project' _LOGGER.error(msg) @@ -804,7 +788,7 @@ def send(self, params: SendParams) -> Squonk2AgentRv: return SuccessRv @synchronized - def ensure_project(self, params: CommonParams) -> Squonk2AgentRv: + def ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: """A blocking method that takes care of the provisioning of the required Squonk2 environment. For Fragalysis this entails the creation of a 'Squonk2 Project' (which also requires a 'Unit' and 'Product'). @@ -812,14 +796,14 @@ def ensure_project(self, params: CommonParams) -> Squonk2AgentRv: If successful the Corresponding Squonk2Project record is returned as the response 'msg' value. """ - assert params - assert isinstance(params, CommonParams) + assert c_params + assert isinstance(c_params, CommonParams) if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' _LOGGER.warning(msg) - # Every public API**MUST** call ping(). + # Every public API **MUST** call ping(). # This ensures Squonk2 is available and gets suitable API tokens... if not self.ping(): msg = 'Squonk2 ping failed.'\ @@ -827,7 +811,11 @@ def ensure_project(self, params: CommonParams) -> Squonk2AgentRv: _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - rv_u: Squonk2AgentRv = self._ensure_project(params) + rv_access: Squonk2AgentRv = self._verify_access(c_params) + if not rv_access.success: + return rv_access + + rv_u: Squonk2AgentRv = self._ensure_project(c_params) if not rv_u.success: msg = 'Failed to create corresponding Squonk2 Project' _LOGGER.error(msg) From 0487c31ee9edd47ecdb7030f3e676153e40a4751 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 2 Feb 2023 23:08:42 +0000 Subject: [PATCH 060/112] Fix typo --- viewer/squonk2_agent.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 1078063d..84062984 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -508,12 +508,10 @@ def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: """ assert c_params assert isinstance(c_params, CommonParams) - - target_access_string = self._get_target_access_string(access_id) + target_access_string = self._get_target_access_string(c_params.access_id) assert target_access_string - # A Squonk2Unit must exist for the Target Access String. rv: Squonk2AgentRv = self._ensure_unit(target_access_string) if not rv.success: @@ -560,9 +558,7 @@ def _verify_access(self, c_params: CommonParams) -> Squonk2AgentRv: # Ensure that the user is allowed to use the given access ID user: User = User.objects.filter(id=c_params.user_id).first() assert user - access_id: str = c_params.access_id - assert access_id - target_access_string = self._get_target_access_string(access_id) + target_access_string = self._get_target_access_string(c_params.access_id) assert target_access_string proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) if not target_access_string in proposal_list: From b1c1291c2cbdd7cb33e69eab3d805141d9ed44b5 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 2 Feb 2023 23:44:36 +0000 Subject: [PATCH 061/112] Fix pylint errors with squonk2_agent --- .pylintrc | 4 ++-- viewer/squonk2_agent.py | 21 ++++++--------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.pylintrc b/.pylintrc index 57b0e145..a92f6a03 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,5 +6,5 @@ # C : Convention warnings # R : Refactoring warnings disable = C, R, - import-error, - django-not-configured + bare-except, + import-error diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 84062984..75a82a12 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -13,7 +13,6 @@ from urllib3.exceptions import InsecureRequestWarning from urllib3 import disable_warnings -from django.core.exceptions import ObjectDoesNotExist from squonk2.auth import Auth from squonk2.as_api import AsApi, AsApiRv from squonk2.dm_api import DmApi, DmApiRv @@ -133,17 +132,9 @@ def __init__(self): # True if configured... self.__configuration_checked: bool = False self.__configured: bool = False - # OIDC hostname and realm. - # Extracted during configuration check from the OIDC variable - self.__oidc_hostname: str = '' - self.__oidc_realm: str = '' # Ignore cert errors? (no) self.__verify_certificates: bool = True - # Set when pre-flight checks have passed. - # When they've been done we can safely (?) continue to use the - # Squonk2 Python client. - self.__pre_flight_check_status: bool = False # The record ID of the Squonk2Org for this deployment. # Set on successful 'pre-flight-check' self.__org_record: Optional[Squonk2Org] = None @@ -392,7 +383,7 @@ def _create_product_and_project(self, assert params # Create an AS Product. - name_truncated, name_full = self._build_product_name(user_name, session_title) + name_truncated, _ = self._build_product_name(user_name, session_title) msg: str = f'Creating AS Product "{name_truncated}"...' _LOGGER.info(msg) @@ -411,7 +402,7 @@ def _create_product_and_project(self, _LOGGER.info(msg) # Create a DM Project - name_truncated, name_full = self._build_project_name(1, 2) + name_truncated, _ = self._build_project_name(1, 2) msg = f'Creating DM Project "{name_truncated}"...' _LOGGER.info(msg) @@ -454,7 +445,7 @@ def _create_product_and_project(self, "sq2_product_uuid": product_uuid} return Squonk2AgentRv(success=True, msg=response_msg) - def _ensure_unit(self, access_id: int) -> Squonk2AgentRv: + def _ensure_unit(self, target_access_string: int) -> Squonk2AgentRv: """Gets or creates a Squonk2 Unit based on a customer's "target access string" (TAS). If a Unit is created its name will begin with the text "Fragalysis " followed by the configured 'SQUONK2_SLUG' (chosen to be unique between all @@ -523,7 +514,7 @@ def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: assert user_name assert session_title - name_truncated, name_full = self._build_product_name(user_name, session_title) + _, name_full = self._build_product_name(user_name, session_title) sq2_project: Optional[Squonk2Project] = Squonk2Project.objects.filter(name=name_full).first() if not sq2_project: msg = f'No existing Squonk2Project for "{name_full}"' @@ -563,7 +554,7 @@ def _verify_access(self, c_params: CommonParams) -> Squonk2AgentRv: proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) if not target_access_string in proposal_list: msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ - f' (access_id={access_id}). Only {proposal_list})' + f' (access_id={c_params.access_id}). Only {proposal_list})' _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -681,7 +672,7 @@ def ping(self) -> Squonk2AgentRv: url = f'{self.__CFG_SQUONK2_DMAPI_URL}/api' try: resp = requests.head(url, verify=self.__verify_certificates) - except Exception as ex: + except Exception: # pylint: disable=broad-except _LOGGER.error('Exception checking DM at %s', url) if resp is None or resp.status_code != 308: msg = f'Squonk2 DM is not responding from {url}' From b31af2a503c8d47dba93722a618fa3013db81439 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 2 Feb 2023 23:50:35 +0000 Subject: [PATCH 062/112] Fix SQ2A typo --- viewer/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viewer/views.py b/viewer/views.py index f96ba0ec..4b303fa7 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3324,7 +3324,7 @@ def list(self, request): return Response(content, status=status.HTTP_403_FORBIDDEN) # Can't use this method if the squonk variables are not set! - sqa_rv = SQ2A.configured() + sqa_rv = _SQ2A.configured() if sqa_rv.success: content = {f'The Squonk2 Agent is not configured ({sqa_rv.msg}'} return Response(content, status=status.HTTP_403_FORBIDDEN) From 0274d89e5a53591927d59149459cbe45057929a9 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 3 Feb 2023 07:55:53 +0000 Subject: [PATCH 063/112] Removed unused imports and import issues (lint) --- viewer/compound_set_upload.py | 13 ------------- viewer/cset_upload.py | 15 +++++++++------ viewer/download_structures.py | 1 - viewer/sdf_check.py | 2 -- viewer/squonk_job_file_transfer.py | 1 - viewer/squonk_job_file_upload.py | 7 ++----- viewer/squonk_job_request.py | 3 +-- viewer/target_set_upload.py | 2 +- viewer/tasks.py | 2 +- viewer/views.py | 2 +- 10 files changed, 15 insertions(+), 33 deletions(-) diff --git a/viewer/compound_set_upload.py b/viewer/compound_set_upload.py index 47b8b8eb..b3460e4d 100644 --- a/viewer/compound_set_upload.py +++ b/viewer/compound_set_upload.py @@ -5,26 +5,14 @@ django.setup() from django.conf import settings -from django.core.files.storage import default_storage -from django.core.files.base import ContentFile - from rdkit import Chem from viewer.models import ( - ComputedMolecule, ScoreDescription, - NumericalScoreValues, - TextScoreValues, Protein, Target, - Molecule, ComputedSetSubmitter) -import ast import os.path -import os - -import psutil - def get_inspiration_frags(cpd, compound_set): pass @@ -115,4 +103,3 @@ def get_additional_mols(filename, compound_set): return f"Missing score descriptions for: {', '.join(missing)}, please re-upload" return mols - diff --git a/viewer/cset_upload.py b/viewer/cset_upload.py index 004f8e31..69cee86b 100644 --- a/viewer/cset_upload.py +++ b/viewer/cset_upload.py @@ -248,6 +248,9 @@ def set_props(self, cpd, props, compound_set): return set_obj def set_mol(self, mol, target, compound_set, filename, zfile=None, zfile_hashvals=None): + # Don't need... + del filename + # zfile = {'zip_obj': zf, 'zf_list': zip_names} print(f'mol: {mol}') smiles = Chem.MolToSmiles(mol) @@ -290,7 +293,7 @@ def set_mol(self, mol, target, compound_set, filename, zfile=None, zfile_hashval insp_frags.append(ref) - orig = mol.GetProp('original SMILES') + _ = mol.GetProp('original SMILES') # Try to get the protein object. # This may fail. @@ -370,10 +373,10 @@ def set_descriptions(self, filename, compound_set): for key in list(description_dict.keys()): if key in descriptions_needed and key not in ['ref_mols', 'ref_pdb', 'index', 'Name', 'original SMILES']: - desc = ScoreDescription.objects.get_or_create(computed_set=compound_set, - name=key, - description=description_dict[key], - )[0] + _ = ScoreDescription.objects.get_or_create(computed_set=compound_set, + name=key, + description=description_dict[key], + ) return mols @@ -423,7 +426,7 @@ def task(self): self.process_mol(mols_to_process[i], self.target, compound_set, sdf_filename, self.zfile, self.zfile_hashvals) # check that molecules have been added to the compound set - check = ComputedMolecule.objects.filter(computed_set=compound_set) + _ = ComputedMolecule.objects.filter(computed_set=compound_set) # check compound set folder exists. cmp_set_folder = os.path.join(settings.MEDIA_ROOT, 'compound_sets') diff --git a/viewer/download_structures.py b/viewer/download_structures.py index 7eb0d92d..d7d358b7 100644 --- a/viewer/download_structures.py +++ b/viewer/download_structures.py @@ -13,7 +13,6 @@ import logging import copy import json -from pathlib import Path import pandoc from django.conf import settings diff --git a/viewer/sdf_check.py b/viewer/sdf_check.py index b7b56e77..95d489aa 100644 --- a/viewer/sdf_check.py +++ b/viewer/sdf_check.py @@ -8,7 +8,6 @@ from rdkit import Chem import validators -import numpy as np from viewer.models import Protein, ComputedSet import datetime @@ -288,4 +287,3 @@ def check_mol_props(mol, validate_dict): validate_dict = missing_field_check(mol, field, validate_dict) return validate_dict - diff --git a/viewer/squonk_job_file_transfer.py b/viewer/squonk_job_file_transfer.py index 1bddf647..30bf27c4 100644 --- a/viewer/squonk_job_file_transfer.py +++ b/viewer/squonk_job_file_transfer.py @@ -7,7 +7,6 @@ from django.conf import settings from rest_framework import status from squonk2.dm_api import DmApi -from rdkit import Chem from celery.utils.log import get_task_logger from viewer.utils import ( diff --git a/viewer/squonk_job_file_upload.py b/viewer/squonk_job_file_upload.py index 2e6521a3..3ea2a8aa 100644 --- a/viewer/squonk_job_file_upload.py +++ b/viewer/squonk_job_file_upload.py @@ -10,20 +10,17 @@ """ import json import os -import shutil -from django.conf import settings -from rest_framework import status from squonk2.dm_api import DmApi from celery.utils.log import get_task_logger -from viewer.models import JobRequest, User +from viewer.models import JobRequest from viewer.utils import ( SDF_VERSION, add_prop_to_sdf, create_media_sub_directory ) -from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent +from viewer.squonk2_agent import Squonk2Agent, get_squonk2_agent logger = get_task_logger(__name__) diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index bcf5b090..f5aeb915 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -10,7 +10,6 @@ import shortuuid -from django.conf import settings from squonk2.dm_api import DmApi from viewer.models import ( Target, @@ -18,7 +17,7 @@ JobRequest, JobFileTransfer ) from viewer.utils import create_squonk_job_request_url, get_https_host -from viewer.squonk2_agent import CommonParams, Squonk2AgentRv, Squonk2Agent, get_squonk2_agent +from viewer.squonk2_agent import CommonParams, Squonk2Agent, get_squonk2_agent logger = logging.getLogger(__name__) diff --git a/viewer/target_set_upload.py b/viewer/target_set_upload.py index d48c87a4..b4d4d0a3 100644 --- a/viewer/target_set_upload.py +++ b/viewer/target_set_upload.py @@ -6,7 +6,7 @@ functions.py """ import logging -import sys, json, os, glob, shutil +import sys, json, os, shutil import datetime from django.contrib.auth.models import User diff --git a/viewer/tasks.py b/viewer/tasks.py index 96c05039..f7083153 100644 --- a/viewer/tasks.py +++ b/viewer/tasks.py @@ -7,7 +7,6 @@ import django django.setup() -from django.conf import settings from celery import shared_task import numpy as np @@ -112,6 +111,7 @@ def process_compound_set(validate_output): process_stage, process_type, validate_dict, validated, params = validate_output logger.info('process_compound_set() ENTER') + logger.info('process_compound_set() process_type=%s', process_type) logger.info('process_compound_set() validated=%s', validated) logger.info('process_compound_set() params=%s', params) diff --git a/viewer/views.py b/viewer/views.py index 4b303fa7..144d4cd7 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -65,7 +65,7 @@ from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent from viewer.squonk2_agent import CommonParams, SendParams, RunJobParams -from .forms import CSetForm, CSetUpdateForm, TSetForm +from .forms import CSetForm, TSetForm from .tasks import ( check_services, erase_compound_set_job_material, From 0e6fd519a0038e54adfd563fd8b15a9512b6baf9 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 3 Feb 2023 07:59:26 +0000 Subject: [PATCH 064/112] Fixed viewer lazy formatting (lint) --- viewer/views.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/viewer/views.py b/viewer/views.py index 144d4cd7..23f60c20 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -1099,10 +1099,10 @@ def email_task_completion(contact_email, message_type, target_name, target_path= 'Please navigate the following link to check the errors: validate_task/' + str(task_id) recipient_list = [contact_email, ] - logger.info('+ email_notify_task_completion email_from: ' + email_from ) - logger.info('+ email_notify_task_completion subject: ' + subject ) - logger.info('+ email_notify_task_completion message: ' + message ) - logger.info('+ email_notify_task_completion contact_email: ' + contact_email ) + logger.info('+ email_notify_task_completion email_from: %s', email_from ) + logger.info('+ email_notify_task_completion subject: %s', subject ) + logger.info('+ email_notify_task_completion message: %s', message ) + logger.info('+ email_notify_task_completion contact_email: %s', contact_email ) # Send email - this should not prevent returning to the screen in the case of error. send_mail(subject, message, email_from, recipient_list, fail_silently=True) @@ -1321,7 +1321,7 @@ def get(self, request, upload_task_id): target_path = '/viewer/target/%s' % target_name response_data['results'] = {} response_data['results']['tset_download_url'] = target_path - logger.info('+ UploadTaskView.get.success -email:'+contact_email) + logger.info('+ UploadTaskView.get.success -email: %s', contact_email) email_task_completion(contact_email, 'upload-success', target_name, target_path=target_path) else: cset_name = results[2] @@ -2395,7 +2395,7 @@ def create(self, request): logger.info('+ DiscoursePostView.post') data = request.data - logger.info('+ DiscoursePostView.post'+json.dumps(data)) + logger.info('+ DiscoursePostView.post %s', json.dumps(data)) if data['category_name'] == '': category_details = None else: @@ -2424,7 +2424,7 @@ def list(self, request): """ logger.info('+ DiscoursePostView.get') query_params = request.query_params - logger.info('+ DiscoursePostView.get'+json.dumps(query_params)) + logger.info('+ DiscoursePostView.get %s', json.dumps(query_params)) discourse_api_key = settings.DISCOURSE_API_KEY @@ -2918,7 +2918,7 @@ def list(self, request): link = DownloadLinks.objects.filter(file_url=file_url) if (link and link[0].zip_file and os.path.isfile(link[0].file_url)): - logger.info('zip_file: {}'.format(link[0].zip_file)) + logger.info('zip_file: %s', link[0].zip_file) # return file and tidy up. file_name = os.path.basename(file_url) From cb8e825f3a31ca0cbff6c0cf6eb27ee2827fcd63 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 3 Feb 2023 08:13:54 +0000 Subject: [PATCH 065/112] Fixed encoding (open) (lint) --- viewer/download_structures.py | 16 ++++++++-------- viewer/serializers.py | 6 +++--- viewer/squonk_job_file_upload.py | 8 ++++---- viewer/target_set_upload.py | 14 +++++++------- viewer/utils.py | 4 ++-- viewer/views.py | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/viewer/download_structures.py b/viewer/download_structures.py index d7d358b7..5fc27b42 100644 --- a/viewer/download_structures.py +++ b/viewer/download_structures.py @@ -109,7 +109,7 @@ def _replace_missing_sdf(molecule, code): # create the file if it doesn't exist... if not os.path.isfile(missing_path): # No file - create one. - with open(missing_path, 'w') as sd_file: + with open(missing_path, 'w', encoding='utf-8') as sd_file: # First line is the protein code, i.e. "PGN_RS02895PGA-x0346_0B" sd_file.write(f'{code}\n') # Now write the lines from the molecule sdf_info record @@ -189,7 +189,7 @@ def _read_and_patch_molecule_name(path, molecule_name=None): # We accumulate the file's content into 'content', # which we eventually return to the caller. content = '' - with open(path, 'r') as f_in: + with open(path, 'r', encoding='utf-8') as f_in: # First line (stripped) first_line = f_in.readline().strip() if first_line: @@ -258,7 +258,7 @@ def _add_file_to_sdf(combined_sdf_file, filepath): fullpath = os.path.join(media_root, filepath) if os.path.isfile(fullpath): - with open(combined_sdf_file, 'a') as f_out: + with open(combined_sdf_file, 'a', encoding='utf-8') as f_out: patched_sdf_content = _read_and_patch_molecule_name(fullpath) f_out.write(patched_sdf_content) return False @@ -320,7 +320,7 @@ def _smiles_files_zip(zip_contents, ziparchive, download_path): """Create and write the smiles file to the ZIP file """ smiles_filename = os.path.join(download_path, 'smiles.smi') - with open(smiles_filename, 'w') as smilesfile: + with open(smiles_filename, 'w', encoding='utf-8') as smilesfile: for smi in zip_contents['molecules']['smiles_info']: smilesfile.write(smi + ',') ziparchive.write( @@ -374,7 +374,7 @@ def _document_file_zip(ziparchive, download_path, original_search, host): readme_filepath = os.path.join(download_path, 'Readme.md') pdf_filepath = os.path.join(download_path, 'Readme.pdf') - with open(readme_filepath, "a") as readme: + with open(readme_filepath, "a", encoding="utf-8") as readme: readme.write("# Documentation for the downloaded zipfile\n") # Download links readme.write("## Download details\n") @@ -394,7 +394,7 @@ def _document_file_zip(ziparchive, download_path, original_search, host): # Download Structure from the template # (but prepare for the template file not existing)? if os.path.isfile(template_file): - with open(template_file, "r") as template: + with open(template_file, "r", encoding="utf-8") as template: readme.write(template.read()) else: logger.warning('Could not find template file (%s)', template_file) @@ -407,7 +407,7 @@ def _document_file_zip(ziparchive, download_path, original_search, host): readme.write('- '+filename+'\n') # Convert markdown to pdf file - doc = pandoc.read(open(readme_filepath, "r").read()) + doc = pandoc.read(open(readme_filepath, "r", encoding="utf-8")).read()) pandoc.write(doc, file=pdf_filepath, format='latex', options=["--columns=72"]) @@ -441,7 +441,7 @@ def _create_structures_zip(target, os.makedirs(download_path, exist_ok=True) error_filename = os.path.join(download_path, "errors.csv") - error_file = open(error_filename, "w") + error_file = open(error_filename, "w", encoding="utf-8") error_file.write("Param,Code,Invalid file reference\n") errors = 0 diff --git a/viewer/serializers.py b/viewer/serializers.py index d77f3480..4d467495 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -345,7 +345,7 @@ class ProtMapInfoSerializer(serializers.ModelSerializer): def get_map_data(self, obj): if obj.map_info: - return open(obj.map_info.path).read() + return open(obj.map_info.path, encoding='utf-8').read() else: return None @@ -359,7 +359,7 @@ class ProtPDBInfoSerializer(serializers.ModelSerializer): pdb_data = serializers.SerializerMethodField() def get_pdb_data(self, obj): - return open(obj.pdb_info.path).read() + return open(obj.pdb_info.path, encoding='utf-8').read() class Meta: @@ -373,7 +373,7 @@ class ProtPDBBoundInfoSerializer(serializers.ModelSerializer): def get_bound_pdb_data(self, obj): if obj.bound_info: - return open(obj.bound_info.path).read() + return open(obj.bound_info.path, encoding='utf-8').read() else: return None diff --git a/viewer/squonk_job_file_upload.py b/viewer/squonk_job_file_upload.py index 3ea2a8aa..6432f21e 100644 --- a/viewer/squonk_job_file_upload.py +++ b/viewer/squonk_job_file_upload.py @@ -69,7 +69,7 @@ def _insert_sdf_blank_mol(job_request, transition_time, sdf_filename): # Do nothing if the first line of the file matches the version we're about to set. blank_present = False - with open(sdf_filename, 'r') as in_file: + with open(sdf_filename, 'r', encoding='utf-8') as in_file: line = in_file.readline() if line and line.startswith(SDF_VERSION): blank_present = True @@ -103,9 +103,9 @@ def _insert_sdf_blank_mol(job_request, transition_time, sdf_filename): 'ref_url': ref_url} blank_mol = _SDF_BLANK_MOL_TEMPLATE.format(**variables) tmp_filename = sdf_filename + '.tmp' - with open(tmp_filename, 'w') as tmp_file: + with open(tmp_filename, 'w', encoding='utf-8') as tmp_file: tmp_file.write(blank_mol) - with open(sdf_filename, 'r') as in_file: + with open(sdf_filename, 'r', encoding='utf-8') as in_file: for line in in_file: tmp_file.write(line) os.remove(sdf_filename) @@ -274,7 +274,7 @@ def process_compound_set_file(jr_id, # We take every field as the key and the description as the value. # If anything goes wrong we erase the SD file and return params = {} - with open(tmp_param_filename, 'r') as param_file: + with open(tmp_param_filename, 'r', encoding='utf-8') as param_file: meta = json.loads(param_file.read()) if 'annotations' not in meta or len(meta['annotations']) == 0: logger.warning('Not processing. No annotations in %s', diff --git a/viewer/target_set_upload.py b/viewer/target_set_upload.py index b4d4d0a3..550b9fc0 100644 --- a/viewer/target_set_upload.py +++ b/viewer/target_set_upload.py @@ -332,7 +332,7 @@ def add_mol(mol_file, prot, projects, lig_id="LIG", chaind_id="Z", """ # create mol object from mol_sd rd_mol = Chem.MolFromMolFile(mol_file) - orig_mol_block = open(mol_file, 'r').read() + orig_mol_block = open(mol_file, 'r', encoding='utf-8')).read() if rd_mol is None: return None @@ -374,7 +374,7 @@ def add_mol(mol_file, prot, projects, lig_id="LIG", chaind_id="Z", if sdf_file: new_mol.sdf_file.save( os.path.basename(sdf_file), - File(open(sdf_file)) + File(open(sdf_file, encoding='utf-8'))) ) new_mol.save() return new_mol @@ -462,7 +462,7 @@ def add_map(new_prot, new_target, map_path, map_type): hotspot_map = HotspotMap.objects.get_or_create( map_type=map_type, target_id=new_target, prot_id=new_prot )[0] - hotspot_map.map_info.save(os.path.basename(map_path), File(open(map_path))) + hotspot_map.map_info.save(os.path.basename(map_path), File(open(map_path, encoding='utf-8')))) return hotspot_map @@ -537,7 +537,7 @@ def remove_not_added(target, xtal_list): def save_confidence(mol, file_path, annotation_type="ligand_confidence"): """save ligand confidence""" - input_dict = json.load(open(file_path)) + input_dict = json.load(open(file_path), encoding='utf-8')) val_store_dict = ["ligand_confidence_comment", "refinement_outcome", "ligand_confidence_int"] for val in val_store_dict: if val in input_dict: @@ -926,7 +926,7 @@ def analyse_target(target, aligned_path): new_frame.sort_values(by='site_name', inplace=True) # one file for new names - with open(os.path.join(aligned_path, 'alternate_names.csv'), 'a') as f: + with open(os.path.join(aligned_path, 'alternate_names.csv'), 'a', encoding='utf-8')) as f: f.write('name,alternate_name\n') for _, row in new_frame.iterrows(): @@ -945,7 +945,7 @@ def analyse_target(target, aligned_path): for i in range(0, len(sorted(unique_sites))): site_mapping[unique_sites[i]] = i - with open(os.path.join(aligned_path, 'hits_ids.csv'), 'a') as f: + with open(os.path.join(aligned_path, 'hits_ids.csv'), 'a', encoding='utf-8')) as f: f.write('crystal_id,site_number\n') for _, row in new_frame.iterrows(): @@ -956,7 +956,7 @@ def analyse_target(target, aligned_path): for crys in list(set([c.code for c in crystal])): f.write(str(crys) + ',' + str(s_id) + '\n') - with open(os.path.join(aligned_path, 'sites.csv'), 'a') as f: + with open(os.path.join(aligned_path, 'sites.csv'), 'a', encoding='utf-8')) as f: f.write('site,id\n') for key in site_mapping.keys(): f.write(str(key) + ',' + str(site_mapping[key]) + '\n') diff --git a/viewer/utils.py b/viewer/utils.py index b01d5c88..f447df19 100644 --- a/viewer/utils.py +++ b/viewer/utils.py @@ -67,8 +67,8 @@ def add_prop_to_sdf(sdf_file_in, sdf_file_out, properties): _REC_SEPARATOR = '$$$$\n' found_separator = False - with open(sdf_file_out, 'a') as sdf_out: - with open(sdf_file_in, 'r') as sdf_in: + with open(sdf_file_out, 'a', encoding='utf-8')) as sdf_out: + with open(sdf_file_in, 'r', encoding='utf-8')) as sdf_in: while True: line = sdf_in.readline() if line: diff --git a/viewer/views.py b/viewer/views.py index 23f60c20..5df7ccf7 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -1478,7 +1478,7 @@ def cset_download(request, name): """ compound_set = ComputedSet.objects.get(unique_name=name) filepath = compound_set.submitted_sdf - with open(filepath.path, 'r') as fp: + with open(filepath.path, 'r', encoding='utf-8')) as fp: data = fp.read() filename = 'compund-set_' + name + '.sdf' response = HttpResponse(content_type='text/plain') @@ -1517,7 +1517,7 @@ def pset_download(request, name): zip_obj = zipfile.ZipFile(buff, 'w') for fp in pdb_filepaths: - data = open(fp, 'r').read() + data = open(fp, 'r', encoding='utf-8')).read() zip_obj.writestr(fp.split('/')[-1], data) zip_obj.close() @@ -2462,7 +2462,7 @@ def create_csv_from_dict(input_dict, title=None, filename=None): if os.path.isfile(download_file): os.remove(download_file) - with open(download_file, "w", newline='') as csvfile: + with open(download_file, "w", newline='', encoding='utf-8') as csvfile: if title: csvfile.write(title) csvfile.write("\n") @@ -2549,7 +2549,7 @@ def list(self, request): file_url = request.GET.get('file_url') if file_url and os.path.isfile(file_url): - with open(file_url) as csvfile: + with open(file_url, encoding='utf8') as csvfile: # return file and tidy up. response = HttpResponse(csvfile, content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=download.csv' From bbdaa00ae2b3b55d3e8cf8be86dcf7eb270affeb Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 3 Feb 2023 08:19:31 +0000 Subject: [PATCH 066/112] Fix encoding typos (last commit) (lint) --- viewer/download_structures.py | 2 +- viewer/target_set_upload.py | 14 +++++++------- viewer/utils.py | 4 ++-- viewer/views.py | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/viewer/download_structures.py b/viewer/download_structures.py index 5fc27b42..7fab42be 100644 --- a/viewer/download_structures.py +++ b/viewer/download_structures.py @@ -407,7 +407,7 @@ def _document_file_zip(ziparchive, download_path, original_search, host): readme.write('- '+filename+'\n') # Convert markdown to pdf file - doc = pandoc.read(open(readme_filepath, "r", encoding="utf-8")).read()) + doc = pandoc.read(open(readme_filepath, "r", encoding="utf-8").read()) pandoc.write(doc, file=pdf_filepath, format='latex', options=["--columns=72"]) diff --git a/viewer/target_set_upload.py b/viewer/target_set_upload.py index 550b9fc0..a5b70b6f 100644 --- a/viewer/target_set_upload.py +++ b/viewer/target_set_upload.py @@ -332,7 +332,7 @@ def add_mol(mol_file, prot, projects, lig_id="LIG", chaind_id="Z", """ # create mol object from mol_sd rd_mol = Chem.MolFromMolFile(mol_file) - orig_mol_block = open(mol_file, 'r', encoding='utf-8')).read() + orig_mol_block = open(mol_file, 'r', encoding='utf-8').read() if rd_mol is None: return None @@ -374,7 +374,7 @@ def add_mol(mol_file, prot, projects, lig_id="LIG", chaind_id="Z", if sdf_file: new_mol.sdf_file.save( os.path.basename(sdf_file), - File(open(sdf_file, encoding='utf-8'))) + File(open(sdf_file, encoding='utf-8')) ) new_mol.save() return new_mol @@ -462,7 +462,7 @@ def add_map(new_prot, new_target, map_path, map_type): hotspot_map = HotspotMap.objects.get_or_create( map_type=map_type, target_id=new_target, prot_id=new_prot )[0] - hotspot_map.map_info.save(os.path.basename(map_path), File(open(map_path, encoding='utf-8')))) + hotspot_map.map_info.save(os.path.basename(map_path), File(open(map_path, encoding='utf-8'))) return hotspot_map @@ -537,7 +537,7 @@ def remove_not_added(target, xtal_list): def save_confidence(mol, file_path, annotation_type="ligand_confidence"): """save ligand confidence""" - input_dict = json.load(open(file_path), encoding='utf-8')) + input_dict = json.load(open(file_path), encoding='utf-8') val_store_dict = ["ligand_confidence_comment", "refinement_outcome", "ligand_confidence_int"] for val in val_store_dict: if val in input_dict: @@ -926,7 +926,7 @@ def analyse_target(target, aligned_path): new_frame.sort_values(by='site_name', inplace=True) # one file for new names - with open(os.path.join(aligned_path, 'alternate_names.csv'), 'a', encoding='utf-8')) as f: + with open(os.path.join(aligned_path, 'alternate_names.csv'), 'a', encoding='utf-8') as f: f.write('name,alternate_name\n') for _, row in new_frame.iterrows(): @@ -945,7 +945,7 @@ def analyse_target(target, aligned_path): for i in range(0, len(sorted(unique_sites))): site_mapping[unique_sites[i]] = i - with open(os.path.join(aligned_path, 'hits_ids.csv'), 'a', encoding='utf-8')) as f: + with open(os.path.join(aligned_path, 'hits_ids.csv'), 'a', encoding='utf-8') as f: f.write('crystal_id,site_number\n') for _, row in new_frame.iterrows(): @@ -956,7 +956,7 @@ def analyse_target(target, aligned_path): for crys in list(set([c.code for c in crystal])): f.write(str(crys) + ',' + str(s_id) + '\n') - with open(os.path.join(aligned_path, 'sites.csv'), 'a', encoding='utf-8')) as f: + with open(os.path.join(aligned_path, 'sites.csv'), 'a', encoding='utf-8') as f: f.write('site,id\n') for key in site_mapping.keys(): f.write(str(key) + ',' + str(site_mapping[key]) + '\n') diff --git a/viewer/utils.py b/viewer/utils.py index f447df19..3471331a 100644 --- a/viewer/utils.py +++ b/viewer/utils.py @@ -67,8 +67,8 @@ def add_prop_to_sdf(sdf_file_in, sdf_file_out, properties): _REC_SEPARATOR = '$$$$\n' found_separator = False - with open(sdf_file_out, 'a', encoding='utf-8')) as sdf_out: - with open(sdf_file_in, 'r', encoding='utf-8')) as sdf_in: + with open(sdf_file_out, 'a', encoding='utf-8') as sdf_out: + with open(sdf_file_in, 'r', encoding='utf-8') as sdf_in: while True: line = sdf_in.readline() if line: diff --git a/viewer/views.py b/viewer/views.py index 5df7ccf7..24b099ff 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -1478,7 +1478,7 @@ def cset_download(request, name): """ compound_set = ComputedSet.objects.get(unique_name=name) filepath = compound_set.submitted_sdf - with open(filepath.path, 'r', encoding='utf-8')) as fp: + with open(filepath.path, 'r', encoding='utf-8') as fp: data = fp.read() filename = 'compund-set_' + name + '.sdf' response = HttpResponse(content_type='text/plain') @@ -1517,7 +1517,7 @@ def pset_download(request, name): zip_obj = zipfile.ZipFile(buff, 'w') for fp in pdb_filepaths: - data = open(fp, 'r', encoding='utf-8')).read() + data = open(fp, 'r', encoding='utf-8').read() zip_obj.writestr(fp.split('/')[-1], data) zip_obj.close() From 3d14f2e36683edeff6ee34476f1df1752814f20b Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 3 Feb 2023 08:22:35 +0000 Subject: [PATCH 067/112] Fixed ValueError (lint) --- viewer/squonk_job_request.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index f5aeb915..ec95384d 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -107,9 +107,8 @@ def create_squonk_job(request): job_transfers = JobFileTransfer.objects.filter(snapshot=snapshot_id) if not job_transfers: logger.warning('No JobFileTransfer object for snapshot %s', snapshot_id) - raise ValueError('No JobFileTransfer object for snapshot %s.' - ' Files must be transferred before a job can be requested.', - snapshot_id) + raise ValueError(f'No JobFileTransfer object for snapshot {snapshot_id}.' + ' Files must be transferred before a job can be requested.') job_transfer = JobFileTransfer.objects.filter(snapshot=snapshot_id).latest('id') if job_transfer.transfer_status != 'SUCCESS': From 004e4eda751b08a2ac840f22de68152c80c472b2 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 3 Feb 2023 08:28:44 +0000 Subject: [PATCH 068/112] Fix unused arguments (lint) --- viewer/compound_set_upload.py | 4 +++- viewer/download_structures.py | 2 ++ viewer/target_set_upload.py | 3 +++ viewer/views.py | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/viewer/compound_set_upload.py b/viewer/compound_set_upload.py index b3460e4d..c01a427d 100644 --- a/viewer/compound_set_upload.py +++ b/viewer/compound_set_upload.py @@ -14,7 +14,9 @@ import os.path def get_inspiration_frags(cpd, compound_set): - pass + # Don't need... + del cpd + del compound_set def process_pdb(pdb_code, target, zfile, zfile_hashvals): diff --git a/viewer/download_structures.py b/viewer/download_structures.py index 7fab42be..c99adfea 100644 --- a/viewer/download_structures.py +++ b/viewer/download_structures.py @@ -367,6 +367,8 @@ def _document_file_zip(ziparchive, download_path, original_search, host): """Create the document file This consists of a template plus an added contents description. """ + # Don't need... + del host template_file = os.path.join("/code/doc_templates", "download_readme_template.md") diff --git a/viewer/target_set_upload.py b/viewer/target_set_upload.py index a5b70b6f..5ae6d34f 100644 --- a/viewer/target_set_upload.py +++ b/viewer/target_set_upload.py @@ -1173,6 +1173,9 @@ def validate_target(new_data_folder, target_name, proposal_ref): :param proposal_ref: A reference to the proposal/visit used for connecting the target to a project/users :return: """ + # Don't need + del proposal_ref + validate_dict = {'Location': [], 'Error': [], 'Line number': []} # Check if there is any data to process diff --git a/viewer/views.py b/viewer/views.py index 24b099ff..a5936d06 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -1939,6 +1939,8 @@ class DSetUploadView(APIView): def put(self, request, format=None): """Method to handle PUT request and upload a design set """ + # Don't need... + del format f = request.FILES['file'] set_type = request.PUT['type'] From 8d340350e870a9bb00c4f655af2e288b99b3a997 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 3 Feb 2023 08:40:24 +0000 Subject: [PATCH 069/112] Adjusted build requirements (lint) Noted duplication of django (requirements) pylint now less noisy pre-commit tweak (unused as-yet) --- .pre-commit-config.yaml | 11 ++++++++--- .pylintrc | 8 ++++++-- build-requirements.txt | 8 +++++--- requirements.txt | 5 +++++ 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d7b95e8a..9f282b84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ --- -minimum_pre_commit_version: 2.20.0 +minimum_pre_commit_version: 2.21.0 exclude: ^(design_docs|doc_templates|docs|tests|) repos: @@ -40,8 +40,13 @@ repos: # the user has done 90% of the lint checks before the code # hits the server. - repo: https://github.com/pycqa/pylint - rev: v2.14.4 + rev: v2.16.1 hooks: - id: pylint args: - - --disable=import-error + - --django-settings-module + - fragalysis.settings + - --load-plugins + - pylint_django + - --load-plugins + - pylint_django.checkers.migrations diff --git a/.pylintrc b/.pylintrc index a92f6a03..e207519c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -5,6 +5,10 @@ # # C : Convention warnings # R : Refactoring warnings -disable = C, R, +disable = C, F, R, bare-except, - import-error + broad-exception-caught, + broad-exception-raised, + fixme, + import-error, + unused-variable diff --git a/build-requirements.txt b/build-requirements.txt index 7413b6a9..93f5d06b 100644 --- a/build-requirements.txt +++ b/build-requirements.txt @@ -1,5 +1,7 @@ # Requirements specifically used to build/test the backend. # These are not rquirements needed by the backend in order for it to run. -pylint -pylint-django -pre-commit == 2.20.0 +pylint == 2.16.1 +pylint-django == 2.5.3 +pre-commit == 2.21.0 + +Django==3.1.14 diff --git a/requirements.txt b/requirements.txt index 15dac38e..a2a12506 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,12 @@ coveralls==2.1.1 cryptography==3.2 decorator==4.4.2 deepdiff==5.8.1 + +# Dango is also referred to in the builtrequirements.txt file +# (for non-execution linting). +# Make sure they're the same! Django==3.1.14 + django-bootstrap3==14.1.0 django-cleanup==5.0.0 django-extensions==3.0.3 From cfaab30c4e79011abd051e2c5576c0996b6965c7 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 15 Feb 2023 12:15:08 +0000 Subject: [PATCH 070/112] Improved logging --- viewer/squonk2_agent.py | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 75a82a12..ff282382 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -330,9 +330,12 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: as_url=self.__CFG_SQUONK2_ASAPI_URL, as_version=as_version) squonk2_org.save() + _LOGGER.info('Created Squonk2Org record for %s', + self.__CFG_SQUONK2_ORG_UUID) else: - _LOGGER.debug('Squonk2Org already exists for %s - nothing to do', - self.__CFG_SQUONK2_ORG_UUID) + _LOGGER.debug('Squonk2Org for %s "%s" already exists - nothing to do', + squonk2_org.uuid, + squonk2_org.name) # Keep the record ID for future use. self.__org_record = squonk2_org @@ -344,7 +347,7 @@ def _delete_as_product(self, product_uuid: str) -> None: """Used in error conditions to remove a previously created Product. If this fails there's nothing else we can do so we just return regardless. """ - _LOGGER.warning('Deleting AS Product %s', product_uuid) + _LOGGER.warning('Deleting AS Product %s...', product_uuid) as_rv: AsApiRv = AsApi.delete_product(self.__org_owner_as_token, product_id=product_uuid) @@ -358,7 +361,7 @@ def _delete_dm_project(self, project_uuid: str) -> None: """Used in error conditions to remove a previously created Project. If this fails there's nothing else we can do so we just return regardless. """ - _LOGGER.warning('Deleting DM Project %s', project_uuid) + _LOGGER.warning('Deleting DM Project %s...', project_uuid) dm_rv: DmApiRv = DmApi.delete_project(self.__org_owner_dm_token, project_id=project_uuid) @@ -384,7 +387,7 @@ def _create_product_and_project(self, # Create an AS Product. name_truncated, _ = self._build_product_name(user_name, session_title) - msg: str = f'Creating AS Product "{name_truncated}"...' + msg: str = f'Creating NEW AS Product "{name_truncated}" (unit={unit.uuid})...' _LOGGER.info(msg) as_rv: AsApiRv = AsApi.create_product(self.__org_owner_as_token, @@ -398,12 +401,12 @@ def _create_product_and_project(self, return Squonk2AgentRv(success=False, msg=msg) product_uuid: str = as_rv.msg['id'] - msg = f'Created AS Product "{product_uuid}"...' + msg = f'Created AS Product {product_uuid}...' _LOGGER.info(msg) # Create a DM Project name_truncated, _ = self._build_project_name(1, 2) - msg = f'Creating DM Project "{name_truncated}"...' + msg = f'Continuing by creating NEW DM Project "{name_truncated}"...' _LOGGER.info(msg) dm_rv: DmApiRv = DmApi.create_project(self.__org_owner_dm_token, @@ -418,7 +421,7 @@ def _create_product_and_project(self, return Squonk2AgentRv(success=False, msg=msg) project_uuid: str = dm_rv.msg["project_id"] - msg = f'Created DM Project "{project_uuid}"...' + msg = f'Created DM Project {project_uuid}...' _LOGGER.info(msg) # Add the user as an Editor to the Project @@ -430,13 +433,14 @@ def _create_product_and_project(self, if not dm_rv.success: msg = f'Failed to add "{user_name}" to DM Project ({dm_rv.msg})' _LOGGER.error(msg) + _LOGGER.warning('Rolling back DM Project and AS Product creation...') # First delete the DM Project amd the corresponding AS Product... self._delete_dm_project(project_uuid) self._delete_as_product(product_uuid) # Then leave... return Squonk2AgentRv(success=False, msg=msg) - msg = f'Added "{user_name} to DM Project {project_uuid} as Editor' + msg = f'"{user_name}" is now an Editor of DM Project {project_uuid}' _LOGGER.info(msg) # If the second call fails - delete the object created in the first @@ -460,11 +464,16 @@ def _ensure_unit(self, target_access_string: int) -> Squonk2AgentRv: unit_name_truncated, unit_name_full = self._build_unit_name(target_access_string) sq2_unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(name=unit_name_full).first() if not sq2_unit: - _LOGGER.info('No existing Unit for "%s"', target_access_string) + _LOGGER.info('No existing Squonk2Unit for "%s"', target_access_string) rv: AsApiRv = AsApi.create_unit(self.__org_owner_as_token, unit_name=unit_name_truncated, org_id=self.__org_record.uuid, billing_day=self.__unit_billing_day) + + _LOGGER.info('Creating NEW Squonk2Unit "%s" (for "%s")', + unit_name_full, + target_access_string) + if not rv.success: msg: str = rv.msg['error'] _LOGGER.error('Failed to create Unit "%s"', target_access_string) @@ -476,10 +485,14 @@ def _ensure_unit(self, target_access_string: int) -> Squonk2AgentRv: organisation_id=self.__org_record.id) sq2_unit.save() - _LOGGER.info('Created NEW Unit %s for "%s"', unit_uuid, target_access_string) + _LOGGER.info('Created Squonk2Unit %s "%s" (for "%s")', + unit_uuid, + unit_name_full, + target_access_string) else: - _LOGGER.debug('Unit %s already exists for "%s" - nothing to do', + _LOGGER.debug('Squonk2Unit %s "%s" already exists (for "%s") - nothing to do', sq2_unit.uuid, + unit_name_full, target_access_string) return Squonk2AgentRv(success=True, msg=sq2_unit) @@ -535,10 +548,10 @@ def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: product_uuid=rv.msg['sq2_product_uuid'], unit_id=unit.id) sq2_project.save() - msg = f'Created NEW Squonk2Project for "{name_full}"' + msg = f'Created NEW Squonk2Project for {sq2_project.uuid} "{name_full}"' _LOGGER.info(msg) else: - msg = f'Project {sq2_project.uuid} already exists for "{name_full}" - nothing to do' + msg = f'Squonk2Project for {sq2_project.uuid} "{name_full}" already exists - nothing to do' _LOGGER.debug(msg) return Squonk2AgentRv(success=True, msg=sq2_project) From 38372f74c47bc2fe9730b4398f97c50841b70bc5 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 15 Feb 2023 12:59:39 +0000 Subject: [PATCH 071/112] Now ignores .vscode directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 502a5088..9bf99724 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ static/ .idea/ +.vscode/ # Byte-compiled / optimized / DLL files __pycache__/ From 32418b91460fe639621914600cf5ea9333f97fe5 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 15 Feb 2023 23:13:53 +0000 Subject: [PATCH 072/112] Project name is the same as the Product --- viewer/squonk2_agent.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index ff282382..7e60b20f 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -210,27 +210,14 @@ def _build_product_name(self, username: str, session_string: str) -> Tuple[str, """Builds a Product name, returning the truncated and un-truncated form""" assert username assert session_string - # AS Products (there's a 1:1 mapping to DM Projects) - # are named using the user and the session + # AS Products are named using the user and the session + # (there's a 1:1 mapping to DM Projects) # The Product name characters are not restricted identifier: str = f'{username}::{session_string}' name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} {identifier}' return name[:_SQ2_MAX_NAME_LENGTH], name - def _build_project_name(self, user_id: int, session_id: int) -> Tuple[str, str]: - assert user_id - assert session_id - # DM Projects (there's a 1:1 mapping to Products) - # are named using the user and the session - - # The Project name characters are RESTRICTED, - # and need to be limited to characters that are - # valid for use with RFC 1123 Label Names - identifier: str = f'{user_id}-{session_id}' - name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} {identifier}' - return name[:_SQ2_MAX_NAME_LENGTH], name - def _get_squonk2_owner_tokens(self) -> Optional[Tuple[str, str]]: """Gets access tokens for the Squonk2 organisation owner. This sets the __keycloak_hostname member and also returns the tokens, @@ -404,8 +391,7 @@ def _create_product_and_project(self, msg = f'Created AS Product {product_uuid}...' _LOGGER.info(msg) - # Create a DM Project - name_truncated, _ = self._build_project_name(1, 2) + # Create a DM Project (using the same name we used for the AS Product) msg = f'Continuing by creating NEW DM Project "{name_truncated}"...' _LOGGER.info(msg) From d7eae9d32ff396115fab7043e31e71864ce301ab Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 16 Feb 2023 15:04:53 +0000 Subject: [PATCH 073/112] Squonk projects now correctly use target title Target transfer always takes place if a user and target has not been seen before --- docker-compose.yml | 2 +- viewer/squonk2_agent.py | 46 ++++++++++++++++++++--------------------- viewer/views.py | 19 ++++++++--------- 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f3f61690..c494c2c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -88,7 +88,7 @@ services: SQUONK2_ASAPI_URL: ${SQUONK2_ASAPI_URL} DUMMY_TAS: ${DUMMY_TAS} DUMMY_USER: ${DUMMY_USER} - DUMMY_SESSION_TITLE: ${DUMMY_SESSION_TITLE} + DUMMY_TARGET_TITLE: ${DUMMY_TARGET_TITLE} ports: - "8080:80" depends_on: diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 7e60b20f..dc03d0db 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -21,7 +21,7 @@ from wrapt import synchronized from api.security import ISpyBSafeQuerySet -from viewer.models import User, SessionProject, Project +from viewer.models import User, Project, Target from viewer.models import Squonk2Project, Squonk2Org, Squonk2Unit _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -118,8 +118,8 @@ def __init__(self): os.environ.get('OIDC_KEYCLOAK_REALM') # Optional config (no '__CFG_' prefix) - self.__DUMMY_SESSION_TITLE: Optional[str] =\ - os.environ.get('DUMMY_SESSION_TITLE') + self.__DUMMY_TARGET_TITLE: Optional[str] =\ + os.environ.get('DUMMY_TARGET_TITLE') self.__DUMMY_USER: Optional[str] =\ os.environ.get('DUMMY_USER') self.__DUMMY_TAS: Optional[str] =\ @@ -183,21 +183,21 @@ def _get_target_access_string(self, access_id: int) -> str: assert target_access_string return target_access_string - def _get_session_title(self, session_id: int) -> str: - # Gets the Session title (if id looks sensible) + def _get_target_title(self, target_id: int) -> str: + # Gets the Target title (if it looks sensible) # or a fixed value (used for testing) - if session_id == 0: + if target_id == 0: assert _TEST_MODE - _LOGGER.warning('Caution - in TEST mode, using __DUMMY_TAS__DUMMY_SESSION_TITLE (%s)', - self.__DUMMY_SESSION_TITLE) - - session_title: str = self.__DUMMY_SESSION_TITLE - if session_id: - session_project: SessionProject = SessionProject.objects.filter(id=session_id).first() - assert session_project - session_title = session_project.title - assert session_title - return session_title + _LOGGER.warning('Caution - in TEST mode, using __DUMMY_TARGET_TITLE (%s)', + self.__DUMMY_TARGET_TITLE) + + target_title: str = self.__DUMMY_TARGET_TITLE + if target_id: + target: Target = Target.objects.filter(id=target_id).first() + assert target + target_title = target.title + assert target_title + return target_title def _build_unit_name(self, target_access_string: str) -> Tuple[str, str]: assert target_access_string @@ -206,15 +206,15 @@ def _build_unit_name(self, target_access_string: str) -> Tuple[str, str]: name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} /{target_access_string}/' return name[:_SQ2_MAX_NAME_LENGTH], name - def _build_product_name(self, username: str, session_string: str) -> Tuple[str, str]: + def _build_product_name(self, username: str, target_title: str) -> Tuple[str, str]: """Builds a Product name, returning the truncated and un-truncated form""" assert username - assert session_string + assert target_title # AS Products are named using the user and the session # (there's a 1:1 mapping to DM Projects) # The Product name characters are not restricted - identifier: str = f'{username}::{session_string}' + identifier: str = f'{username}::{target_title}' name: str = f'{_SQ2_NAME_PREFIX} {self.__CFG_SQUONK2_SLUG} {identifier}' return name[:_SQ2_MAX_NAME_LENGTH], name @@ -509,18 +509,18 @@ def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: unit: Squonk2Unit = rv.msg user_name: str = self._get_user_name(c_params.user_id) - session_title: str = self._get_session_title(c_params.session_id) + target_title: str = self._get_target_title(c_params.target_id) assert user_name - assert session_title + assert target_title - _, name_full = self._build_product_name(user_name, session_title) + _, name_full = self._build_product_name(user_name, target_title) sq2_project: Optional[Squonk2Project] = Squonk2Project.objects.filter(name=name_full).first() if not sq2_project: msg = f'No existing Squonk2Project for "{name_full}"' _LOGGER.info(msg) # Need to call upon Squonk2 to create a 'Product' # (and corresponding 'Product'). - rv = self._create_product_and_project(unit, user_name, session_title, c_params) + rv = self._create_product_and_project(unit, user_name, target_title, c_params) if not rv.success: msg = f'Failed creating AS Product or DM Project ({rv.msg})' _LOGGER.error(msg) diff --git a/viewer/views.py b/viewer/views.py index a5936d06..540d310d 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3167,20 +3167,19 @@ def create(self, request): if error: return Response(error['message'], status=error['status']) - # If transfer has already happened find the latest - job_transfers = JobFileTransfer.objects.filter(snapshot=snapshot_id) + # If transfer has already happened find the latest. + # + # Squonk Access Control feature creates a 'Project' for + # each unique combination of user and target. + # So if there's a transfer for user 'a', and taregt 'b' + # then we can assume the transfer has taken place + # and we do not need to start another. + job_transfers = JobFileTransfer.objects.filter(target=target_id, + user=user.id) if job_transfers: job_transfer = job_transfers.latest('id') else: job_transfer = None - if job_transfer and not job_transfer.target: - msg = f'JobTransfer record ({job_transfer.id})' \ - f' for snapshot {snapshot_id} has no target.' \ - ' Cannot continue' - content = {'message': msg} - logger.error(msg) - return Response(content, - status=status.HTTP_404_NOT_FOUND) # The root (in the Squonk project) where files will be written. # This is "understood" by the celery task (which uses this constant). From d77708b66782c88b5ad315c81dd13a82a0b48332 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 16 Feb 2023 15:47:38 +0000 Subject: [PATCH 074/112] Every JobTransfer Post now results in a new Job Transfer record --- viewer/views.py | 117 +++++++++++++++--------------------------------- 1 file changed, 36 insertions(+), 81 deletions(-) diff --git a/viewer/views.py b/viewer/views.py index 540d310d..ec1aab9e 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3167,94 +3167,49 @@ def create(self, request): if error: return Response(error['message'], status=error['status']) - # If transfer has already happened find the latest. - # - # Squonk Access Control feature creates a 'Project' for - # each unique combination of user and target. - # So if there's a transfer for user 'a', and taregt 'b' - # then we can assume the transfer has taken place - # and we do not need to start another. - job_transfers = JobFileTransfer.objects.filter(target=target_id, - user=user.id) - if job_transfers: - job_transfer = job_transfers.latest('id') - else: - job_transfer = None - # The root (in the Squonk project) where files will be written. # This is "understood" by the celery task (which uses this constant). # e.g. 'fragalysis-files' transfer_root = settings.SQUONK2_MEDIA_DIRECTORY logger.info('+ transfer_root=%s', transfer_root) - if job_transfer: - - # A pre-existing transfer... - transfer_target = job_transfer.target.title - if (job_transfer.transfer_status == 'PENDING' or - job_transfer.transfer_status == 'STARTED'): - - logger.info('+ Existing transfer_status=%s', job_transfer.transfer_status) - content = {'transfer_root': transfer_root, - 'transfer_target': transfer_target, - 'message': 'Files currently being transferred'} - return Response(content, - status=status.HTTP_208_ALREADY_REPORTED) - - if (target.upload_datetime and job_transfer.transfer_datetime) \ - and target.upload_datetime < job_transfer.transfer_datetime: - - # The target data has already been transferred for the snapshot. - logger.info('+ Existing transfer finished (transfer_status=%s)', - job_transfer.transfer_status) - content = {'transfer_root': transfer_root, - 'transfer_target': transfer_target, - 'message': 'Files already transferred for this job'} - return Response(content, - status=status.HTTP_200_OK) - - # Restart existing transfer - it must have failed or be outdated - job_transfer.user = request.user - - else: - - # Create new file transfer job - logger.info('+ Calling ensure_project() to get the Squonk2 Project...') - - # This requires a Squonk2 Project (created by the Squonk2Agent). - # It may be an existing project, or it might be a new project. - common_params = CommonParams(user_id=user.id, - access_id=access_id, - target_id=target_id, - session_id=session_project_id) - sq2_rv = _SQ2A.ensure_project(common_params) - if not sq2_rv.success: - msg = f'Failed to get/create a Squonk2 Project' \ - f' for User "{user.username}", Access ID {access_id},' \ - f' Target ID {target_id}, and SessionProject ID {session_project_id}.' \ - f' Got "{sq2_rv.msg}".' \ - ' Cannot continue' - content = {'message': msg} - logger.error(msg) - return Response(content, status=status.HTTP_404_NOT_FOUND) + # Create new file transfer job + logger.info('+ Calling ensure_project() to get the Squonk2 Project...') + + # This requires a Squonk2 Project (created by the Squonk2Agent). + # It may be an existing project, or it might be a new project. + common_params = CommonParams(user_id=user.id, + access_id=access_id, + target_id=target_id, + session_id=session_project_id) + sq2_rv = _SQ2A.ensure_project(common_params) + if not sq2_rv.success: + msg = f'Failed to get/create a Squonk2 Project' \ + f' for User "{user.username}", Access ID {access_id},' \ + f' Target ID {target_id}, and SessionProject ID {session_project_id}.' \ + f' Got "{sq2_rv.msg}".' \ + ' Cannot continue' + content = {'message': msg} + logger.error(msg) + return Response(content, status=status.HTTP_404_NOT_FOUND) - # The Squonk2Project record is in the response msg - squonk2_project_uuid = sq2_rv.msg.uuid - squonk2_unit_name = sq2_rv.msg.unit.name - squonk2_unit_uuid = sq2_rv.msg.unit.uuid - logger.info('+ ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)', - squonk2_project_uuid, squonk2_unit_name, squonk2_unit_uuid) - - job_transfer = JobFileTransfer() - job_transfer.user = request.user - job_transfer.proteins = [p['code'] for p in proteins] - job_transfer.compounds = [c['name'] for c in compounds] - # We should use a foreign key, - # but to avoid migration issues with the existing code - # we continue to use the project UUID string field. - job_transfer.squonk_project = squonk2_project_uuid - job_transfer.target = Target.objects.get(id=target_id) - job_transfer.snapshot = Snapshot.objects.get(id=snapshot_id) + # The Squonk2Project record is in the response msg + squonk2_project_uuid = sq2_rv.msg.uuid + squonk2_unit_name = sq2_rv.msg.unit.name + squonk2_unit_uuid = sq2_rv.msg.unit.uuid + logger.info('+ ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)', + squonk2_project_uuid, squonk2_unit_name, squonk2_unit_uuid) + + job_transfer = JobFileTransfer() + job_transfer.user = request.user + job_transfer.proteins = [p['code'] for p in proteins] + job_transfer.compounds = [c['name'] for c in compounds] + # We should use a foreign key, + # but to avoid migration issues with the existing code + # we continue to use the project UUID string field. + job_transfer.squonk_project = squonk2_project_uuid + job_transfer.target = Target.objects.get(id=target_id) + job_transfer.snapshot = Snapshot.objects.get(id=snapshot_id) # The 'transfer target' (a sub-directory of the transfer root) # For example the root might be 'fragalysis-files' From 7c88e6205810522ebc89fa0645c861d06949ed16 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 21 Feb 2023 14:29:47 +0000 Subject: [PATCH 075/112] fix: Should now prefix new Project titles with a code Also sets public project value (if no users) --- viewer/target_set_upload.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/viewer/target_set_upload.py b/viewer/target_set_upload.py index 5ae6d34f..340f4bf5 100644 --- a/viewer/target_set_upload.py +++ b/viewer/target_set_upload.py @@ -477,7 +477,7 @@ def delete_users(project): project.save() -def get_create_projects(target, proposal_ref): +def get_create_projects(target, proposal_ref, proposal_code='lb'): """Add proposals and visits as projects for a given target. :param new_target: the target being added @@ -494,6 +494,10 @@ def get_create_projects(target, proposal_ref): # The first word is the ISPY proposal/visit name that is used as the title of the project. # It can be set to OPEN in which case there are no users. visit = proposal_ref.split()[0] + # If the visit is not prefixed by the proposal code + # (typically a 2-letter sequence like "lb") then prefix it. + if visit[0].isdigit(): + visit = f"{proposal_code}{visit}" project = Project.objects.get_or_create(title=visit)[0] projects.append(project) @@ -504,9 +508,14 @@ def get_create_projects(target, proposal_ref): target.project_id.add(project) # Remaining words in proposal_ref (if any) must be fedid's which are used to find users information. + num_users = 0 for fedid in proposal_ref.split()[1:]: user = User.objects.get_or_create(username=fedid, password="")[0] project.user_id.add(user) + num_users += 1 + if num_users == 0: + project.open_to_public = True + target.upload_progess = 10.00 target.save() From 00543636a6f17a754c8b63696fee7c517f0301bd Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 22 Feb 2023 15:05:42 +0000 Subject: [PATCH 076/112] Fixes _verify_access Now checks whether Project is public before checking ISPyB --- viewer/squonk2_agent.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index dc03d0db..c2e958f8 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -545,15 +545,27 @@ def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: def _verify_access(self, c_params: CommonParams) -> Squonk2AgentRv: """Checks the user has access to the project. """ - # Ensure that the user is allowed to use the given access ID + access_id: int = c_params.access_id + assert access_id + project: Optional[Project] = Project.objects.filter(id=access_id).first() + if not project: + msg = f'Access ID (Project) {access_id} does not exist' + _LOGGER.warning(msg) + return Squonk2AgentRv(success=False, msg=msg) + # Public projects can always be accessed... + if project.open_to_public: + return SuccessRv + + # The project is not a public Project, + # ensure that the user is allowed to use the given access ID user: User = User.objects.filter(id=c_params.user_id).first() assert user - target_access_string = self._get_target_access_string(c_params.access_id) + target_access_string = self._get_target_access_string(access_id) assert target_access_string proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user(user) if not target_access_string in proposal_list: msg = f'The user ({user.username}) cannot access "{target_access_string}"' \ - f' (access_id={c_params.access_id}). Only {proposal_list})' + f' (access_id={access_id}). Only {proposal_list})' _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) From 321949bc2b90ce7961c5b8ca7c262e76303ff841 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 23 Feb 2023 07:29:18 +0000 Subject: [PATCH 077/112] Better connection exception handling --- api/security.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/api/security.py b/api/security.py index bd2c73e3..6c399a3d 100644 --- a/api/security.py +++ b/api/security.py @@ -66,9 +66,9 @@ def get_remote_conn(): conn = None try: conn = SSHConnector(**ispyb_credentials) - except ValueError as cex: - logger.info("ssh_credentials=%s", ssh_credentials) - logger.error("Got ValueError exception getting SSH connection (%s)", cex) + except: + logger.info("ispyb_credentials=%s", ispyb_credentials) + logger.exception("Exception creating SSHConnector") return conn @@ -89,7 +89,13 @@ def get_conn(): if not credentials["host"]: return None - conn = Connector(**credentials) + conn = None + try: + conn = Connector(**credentials) + except: + logger.info("credentials=%s", credentials) + logger.exception("Exception creating Connector") + return conn From 288d4fa3128d96884638de7e25b1ea10949d46d1 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 1 Mar 2023 12:49:45 +0000 Subject: [PATCH 078/112] Squonk projects are now private --- requirements.txt | 2 +- viewer/squonk2_agent.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a2a12506..2dbdbacf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.17.1 +im-squonk2-client==1.17.6 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index c2e958f8..b72bf94f 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -397,6 +397,7 @@ def _create_product_and_project(self, dm_rv: DmApiRv = DmApi.create_project(self.__org_owner_dm_token, project_name=name_truncated, + private=True, as_tier_product_id=product_uuid) if not dm_rv.success: msg = f'Failed to create DM Project ({dm_rv.msg})' From c9136d222eb05bd77134de45368b3923bfbe1a8e Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 1 Mar 2023 14:21:29 +0000 Subject: [PATCH 079/112] Can use Squonk unless part of the project/visit --- viewer/squonk2_agent.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index b72bf94f..71d20513 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -553,12 +553,10 @@ def _verify_access(self, c_params: CommonParams) -> Squonk2AgentRv: msg = f'Access ID (Project) {access_id} does not exist' _LOGGER.warning(msg) return Squonk2AgentRv(success=False, msg=msg) - # Public projects can always be accessed... - if project.open_to_public: - return SuccessRv - # The project is not a public Project, - # ensure that the user is allowed to use the given access ID + # Ensure that the user is allowed to use the given access ID. + # Even on public projects the user must be part of the project + # to use Squonk. user: User = User.objects.filter(id=c_params.user_id).first() assert user target_access_string = self._get_target_access_string(access_id) From 9bd9b84cb9d9404593bdcc52da6d63b73152a041 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 3 Mar 2023 16:06:21 +0000 Subject: [PATCH 080/112] Use of Squonk Python client 1.18.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2dbdbacf..191f725c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.17.6 +im-squonk2-client==1.18.0 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 From bd33d0fa275604a0fa1bebe8f6fcd45f5b96db10 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 7 Mar 2023 12:00:07 +0000 Subject: [PATCH 081/112] Initial work on job_access endpoint - Job Request has new project reference - Squonk2Agent now offers observer access to projects - New job_access endpoint --- api/urls.py | 1 + .../0028_add_job_request_project.py | 19 ++++ viewer/models.py | 1 + viewer/squonk2_agent.py | 34 +++++++ viewer/squonk_job_request.py | 4 +- viewer/views.py | 98 ++++++++++++++++++- 6 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 viewer/migrations/0028_add_job_request_project.py diff --git a/api/urls.py b/api/urls.py index 8ce84069..7debff47 100644 --- a/api/urls.py +++ b/api/urls.py @@ -88,6 +88,7 @@ router.register(r"job_request", viewer_views.JobRequestView, basename='job_request') router.register(r"job_callback", viewer_views.JobCallBackView, basename='job_callback') router.register(r"job_config", viewer_views.JobConfigView, basename='job_config') +router.register(r"job_access", viewer_views.JobAccessView, basename='job_access') from rest_framework_swagger.renderers import OpenAPIRenderer, SwaggerUIRenderer from rest_framework.decorators import api_view, renderer_classes diff --git a/viewer/migrations/0028_add_job_request_project.py b/viewer/migrations/0028_add_job_request_project.py new file mode 100644 index 00000000..6e05e615 --- /dev/null +++ b/viewer/migrations/0028_add_job_request_project.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.14 on 2023-03-07 11:46 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0027_sessionproject_project'), + ] + + operations = [ + migrations.AddField( + model_name='jobrequest', + name='project', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='viewer.project'), + ), + ] diff --git a/viewer/models.py b/viewer/models.py index dc2c3a0a..c102b131 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -1180,6 +1180,7 @@ class JobRequest(models.Model): snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE) target = models.ForeignKey(Target, null=True, on_delete=models.CASCADE, db_index=True) + project = models.ForeignKey(Project, null=True, on_delete=models.CASCADE) squonk_project = models.CharField(max_length=200, null=True) squonk_job_spec = models.JSONField(encoder=DjangoJSONEncoder, null=True) # Start and finish times for the Job diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 71d20513..6a3a7ab0 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -51,6 +51,10 @@ SendParams: namedtuple = namedtuple("Send", ["common", "snapshot_id"]) +# Parameters for the grant_access() method. +AccessParams: namedtuple = namedtuple("Access", ["username", + "project_uuid"]) + _SUPPORTED_PRODUCT_FLAVOURS: List[str] = ["BRONZE", "SILVER", "GOLD"] # Squonk2 Have defined limits - assumed here. @@ -820,6 +824,36 @@ def ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: return rv_u + @synchronized + def grant_access(self, a_params: AccessParams) -> Squonk2AgentRv: + """A blocking method that takes care of sending a set of files to + the configured Squonk2 installation. + """ + assert a_params + assert isinstance(a_params, AccessParams) + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API **MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + dm_rv: DmApiRv = DmApi.add_project_observer(self.__org_owner_dm_token, + project_id=a_params.project_uuid, + observer=a_params.username) + if not dm_rv.success: + msg = f'Failed to add DM Project Observer ({dm_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + return SuccessRv + # A placeholder for the Agent object _AGENT_SINGLETON: Optional[Squonk2Agent] = None diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index ec95384d..cb74c318 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -12,7 +12,8 @@ from squonk2.dm_api import DmApi -from viewer.models import ( Target, +from viewer.models import ( Project, + Target, Snapshot, JobRequest, JobFileTransfer ) @@ -145,6 +146,7 @@ def create_squonk_job(request): job_request = JobRequest() job_request.squonk_job_name = squonk_job_name job_request.user = request.user + job_request.project = Project.objects.get(id=access_id) job_request.snapshot = Snapshot.objects.get(id=snapshot_id) job_request.target = Target.objects.get(id=target_id) # We should use a foreign key, diff --git a/viewer/views.py b/viewer/views.py index ec1aab9e..7ec44e33 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -63,7 +63,7 @@ ) from viewer import filters from viewer.squonk2_agent import Squonk2AgentRv, Squonk2Agent, get_squonk2_agent -from viewer.squonk2_agent import CommonParams, SendParams, RunJobParams +from viewer.squonk2_agent import AccessParams, CommonParams, SendParams, RunJobParams from .forms import CSetForm, TSetForm from .tasks import ( @@ -3384,7 +3384,7 @@ def create(self, request): target_id = request.data['target'] snapshot_id = request.data['snapshot'] session_project_id = request.data['session_project'] - access_id = request.data['access'] # The access ID/legacy Project record title + access_id = request.data['access'] # The access ID/legacy Project record logger.info('+ user="%s" (id=%s)', user.username, user.id) logger.info('+ access_id=%s', access_id) @@ -3603,3 +3603,97 @@ def update(self, request, code=None): jr.id, task_upload) return HttpResponse(status=204) + +class JobAccessView(viewsets.ReadOnlyModelViewSet): + """Django view that calls Squonk to allow a user (who is able to see a Job) + the ability to access that Job in Squonk. To be successful the user + must have access to the corresponding Fragalysis Project. + + Methods + ------- + allowed requests: + - GET: Get job access + + url: + api/job_access + get params: + - job_request_id: The identity of the JobRequest (the Job) the user needs access to + + Returns: A structure with 'accessible' set to True on success. On failure + an 'error' property contains a reason (string) + + example input for get + + .. code-block:: + + /api/job_access/?job_request_id=17 + """ + def list(self, request): + """Method to handle GET request + """ + query_params = request.query_params + logger.info('+ JobAccessView.get: %s', json.dumps(query_params)) + + err_response = {'accessible': False} + ok_response = {'accessible': True, 'error': ''} + + # Only authenticated users can have squonk jobs + user = self.request.user + if not user.is_authenticated: + content = {'accessible': False, + 'error': 'Only authenticated users can access Squonk Jobs'} + return Response(content, status=status.HTTP_403_FORBIDDEN) + + # Can't use this method if the squonk variables are not set! + sqa_rv = _SQ2A.configured() + if not sqa_rv.success: + err_response['error'] = f'The Squonk2 Agent is not configured ({sqa_rv.msg}' + return Response(err_response, status=status.HTTP_403_FORBIDDEN) + + # Get the JobRequest Record + jr_id = request.query_params.get('job_request_id', None) + if not jr_id or jr_id < 1: + err_response['error'] = f'The JobRequest ID is invalid ({jr_id}' + return Response(err_response, status=status.HTTP_400_BAD_REQUEST) + + jr_list = JobRequest.objects.filter(id=jr_id) + if len(jr_list) == 0: + err_response['error'] = f'The JobRequest does not exist ({jr_id}' + return Response(err_response, status=status.HTTP_400_BAD_REQUEST) + jr = jr_list[0] + + # JobRequest must have a Squonk Project value + if not jr.squonk_project: + err_response['error'] = f'The JobRequest has no Squonk2Project value ({jr_id}' + return Response(err_response, status=status.HTTP_403_FORBIDDEN) + + # User must have access to the Job's Project. + # If there is no Project (legacy records) we skip this check + # and simply grabt the user access. + if jr.project and jr.project.title: + # The project title is the Job access string. + # To check access we need this and the User's ID + access_id = jr.project.title + sq2a_common_params: CommonParams = CommonParams(user_id=user.id, + access_id=access_id, + session_id=None, + target_id=None) + sq2a_run_job_params: RunJobParams = RunJobParams(common=sq2a_common_params, + job_spec=None, + callback_url=None) + sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params) + if not sq2a_rv.success: + err_response['error'] = f'Access to the Job ({access_id}) is denied. {sq2a_rv.msg}' + return Response(err_response, status=status.HTTP_403_FORBIDDEN) + + # Success - try to grant access to the user + sq2a_access_params: AccessParams = AccessParams(username=user.username, + project_uuid=jr.squonk_project) + sqa_rv = _SQ2A.grant_access(sq2a_access_params) + if not sqa_rv.success: + err_response['error'] = f'The Squonk2 Agent failed to grant access ({sqa_rv.msg}' + logger.warning('+ JobAccessView.get: error=%s', content['error']) + return Response(err_response, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + logger.info('+ JobAccessView.get: Success for %s/%s', user.username, jr_id) + return Response(ok_response) From a03e29c47a27baecbc7f16792935b500fe6cdb07 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 7 Mar 2023 13:19:46 +0000 Subject: [PATCH 082/112] Grant access should now work for Job owner --- viewer/views.py | 70 ++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/viewer/views.py b/viewer/views.py index 7ec44e33..c4749c8a 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3607,7 +3607,8 @@ def update(self, request, code=None): class JobAccessView(viewsets.ReadOnlyModelViewSet): """Django view that calls Squonk to allow a user (who is able to see a Job) the ability to access that Job in Squonk. To be successful the user - must have access to the corresponding Fragalysis Project. + must have access to the corresponding Fragalysis Project. This can be called by + the Job 'owner', who always has access. Methods ------- @@ -3632,7 +3633,7 @@ def list(self, request): """Method to handle GET request """ query_params = request.query_params - logger.info('+ JobAccessView.get: %s', json.dumps(query_params)) + logger.info('+ JobAccessView/GET %s', json.dumps(query_params)) err_response = {'accessible': False} ok_response = {'accessible': True, 'error': ''} @@ -3668,32 +3669,41 @@ def list(self, request): return Response(err_response, status=status.HTTP_403_FORBIDDEN) # User must have access to the Job's Project. - # If there is no Project (legacy records) we skip this check - # and simply grabt the user access. - if jr.project and jr.project.title: - # The project title is the Job access string. - # To check access we need this and the User's ID - access_id = jr.project.title - sq2a_common_params: CommonParams = CommonParams(user_id=user.id, - access_id=access_id, - session_id=None, - target_id=None) - sq2a_run_job_params: RunJobParams = RunJobParams(common=sq2a_common_params, - job_spec=None, - callback_url=None) - sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params) - if not sq2a_rv.success: - err_response['error'] = f'Access to the Job ({access_id}) is denied. {sq2a_rv.msg}' - return Response(err_response, status=status.HTTP_403_FORBIDDEN) - - # Success - try to grant access to the user - sq2a_access_params: AccessParams = AccessParams(username=user.username, - project_uuid=jr.squonk_project) - sqa_rv = _SQ2A.grant_access(sq2a_access_params) - if not sqa_rv.success: - err_response['error'] = f'The Squonk2 Agent failed to grant access ({sqa_rv.msg}' - logger.warning('+ JobAccessView.get: error=%s', content['error']) - return Response(err_response, status=status.HTTP_500_INTERNAL_SERVER_ERROR) - - logger.info('+ JobAccessView.get: Success for %s/%s', user.username, jr_id) + # If the user is not the owner of the Job, and there is a Project, + # we then check the user has access to the gievn access ID. + # + # If the Job has no Project (Jobs created before this chnage will not have a Project) + # or the user is the owner of the Job we skip this check. + if user.id != jr.user.id: + logger.info('+ JobAccessView/GET Checking access to JobRequest %s for "%s"', + jr_id, user.username) + if not jr.project or not jr.project.title: + logger.warning('+ JobAccessView/GET No Project (or title) for JobRequest %s - granting acess', + jr_id) + else: + # The project title is the Job access string. + # To check access we need this and the User's ID + access_id = jr.project.title + sq2a_common_params: CommonParams = CommonParams(user_id=user.id, + access_id=access_id, + session_id=None, + target_id=None) + sq2a_run_job_params: RunJobParams = RunJobParams(common=sq2a_common_params, + job_spec=None, + callback_url=None) + sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params) + if not sq2a_rv.success: + err_response['error'] = f'Access to the Job ({access_id}) is denied. {sq2a_rv.msg}' + return Response(err_response, status=status.HTTP_403_FORBIDDEN) + + # It's not the Job owner, so grant access for this user + sq2a_access_params: AccessParams = AccessParams(username=user.username, + project_uuid=jr.squonk_project) + sqa_rv = _SQ2A.grant_access(sq2a_access_params) + if not sqa_rv.success: + err_response['error'] = f'The Squonk2 Agent failed to grant access ({sqa_rv.msg}' + logger.warning('+ JobAccessView/GET error=%s', content['error']) + return Response(err_response, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + logger.info('+ JobAccessView/GET Success for %s/"%s"', jr_id, user.username) return Response(ok_response) From 85bfc0023766de0454e17011194e8777ee444461 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 7 Mar 2023 15:54:58 +0000 Subject: [PATCH 083/112] JobAccess now a basic view (at viewer/job_access) --- api/urls.py | 1 - viewer/urls.py | 1 + viewer/views.py | 32 +++++++++++++++++--------------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/api/urls.py b/api/urls.py index 7debff47..8ce84069 100644 --- a/api/urls.py +++ b/api/urls.py @@ -88,7 +88,6 @@ router.register(r"job_request", viewer_views.JobRequestView, basename='job_request') router.register(r"job_callback", viewer_views.JobCallBackView, basename='job_callback') router.register(r"job_config", viewer_views.JobConfigView, basename='job_config') -router.register(r"job_access", viewer_views.JobAccessView, basename='job_access') from rest_framework_swagger.renderers import OpenAPIRenderer, SwaggerUIRenderer from rest_framework.decorators import api_view, renderer_classes diff --git a/viewer/urls.py b/viewer/urls.py index 1a0c3b1c..ce8a84c4 100644 --- a/viewer/urls.py +++ b/viewer/urls.py @@ -17,4 +17,5 @@ url(r'^protein_set/(?P.+)/$', views.pset_download, name='protein_set'), url(r'^target/(?P.+)/$', views.tset_download, name='target_set'), url(r'upload_designs/', views.DSetUploadView.as_view(), name='upload_designs'), + url(r"job_access/", views.JobAccessView.as_view(), name='job_access'), ] diff --git a/viewer/views.py b/viewer/views.py index c4749c8a..9a981cd1 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3604,7 +3604,7 @@ def update(self, request, code=None): return HttpResponse(status=204) -class JobAccessView(viewsets.ReadOnlyModelViewSet): +class JobAccessView(APIView): """Django view that calls Squonk to allow a user (who is able to see a Job) the ability to access that Job in Squonk. To be successful the user must have access to the corresponding Fragalysis Project. This can be called by @@ -3629,7 +3629,7 @@ class JobAccessView(viewsets.ReadOnlyModelViewSet): /api/job_access/?job_request_id=17 """ - def list(self, request): + def get(self, request): """Method to handle GET request """ query_params = request.query_params @@ -3648,24 +3648,24 @@ def list(self, request): # Can't use this method if the squonk variables are not set! sqa_rv = _SQ2A.configured() if not sqa_rv.success: - err_response['error'] = f'The Squonk2 Agent is not configured ({sqa_rv.msg}' + err_response['error'] = f'Squonk is not configured for this stack' return Response(err_response, status=status.HTTP_403_FORBIDDEN) # Get the JobRequest Record jr_id = request.query_params.get('job_request_id', None) if not jr_id or jr_id < 1: - err_response['error'] = f'The JobRequest ID is invalid ({jr_id}' + err_response['error'] = f'The JobRequest ID ("{jr_id}") is invalid' return Response(err_response, status=status.HTTP_400_BAD_REQUEST) jr_list = JobRequest.objects.filter(id=jr_id) if len(jr_list) == 0: - err_response['error'] = f'The JobRequest does not exist ({jr_id}' + err_response['error'] = f'The JobRequest does not exist' return Response(err_response, status=status.HTTP_400_BAD_REQUEST) jr = jr_list[0] # JobRequest must have a Squonk Project value if not jr.squonk_project: - err_response['error'] = f'The JobRequest has no Squonk2Project value ({jr_id}' + err_response['error'] = f'The JobRequest has no Squonk Project value ({jr_id})' return Response(err_response, status=status.HTTP_403_FORBIDDEN) # User must have access to the Job's Project. @@ -3675,10 +3675,11 @@ def list(self, request): # If the Job has no Project (Jobs created before this chnage will not have a Project) # or the user is the owner of the Job we skip this check. if user.id != jr.user.id: - logger.info('+ JobAccessView/GET Checking access to JobRequest %s for "%s"', - jr_id, user.username) + logger.info('+ JobAccessView/GET Checking access to JobRequest %s for "%s" (%s)', + jr_id, user.username, jr.squonk_project) if not jr.project or not jr.project.title: - logger.warning('+ JobAccessView/GET No Project (or title) for JobRequest %s - granting acess', + logger.warning('+ JobAccessView/GET No Fragalysis Project (or title)' + ' for JobRequest %s - granting access', jr_id) else: # The project title is the Job access string. @@ -3689,21 +3690,22 @@ def list(self, request): session_id=None, target_id=None) sq2a_run_job_params: RunJobParams = RunJobParams(common=sq2a_common_params, - job_spec=None, - callback_url=None) + job_spec=None, + callback_url=None) sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params) if not sq2a_rv.success: - err_response['error'] = f'Access to the Job ({access_id}) is denied. {sq2a_rv.msg}' + err_response['error'] = f'Access to the Job for {access_id} is denied. {sq2a_rv.msg}' return Response(err_response, status=status.HTTP_403_FORBIDDEN) - # It's not the Job owner, so grant access for this user + # All checks passed ... grant access for this user sq2a_access_params: AccessParams = AccessParams(username=user.username, project_uuid=jr.squonk_project) sqa_rv = _SQ2A.grant_access(sq2a_access_params) if not sqa_rv.success: - err_response['error'] = f'The Squonk2 Agent failed to grant access ({sqa_rv.msg}' + err_response['error'] = f'Squonk failed to grant access ({sqa_rv.msg})' logger.warning('+ JobAccessView/GET error=%s', content['error']) return Response(err_response, status=status.HTTP_500_INTERNAL_SERVER_ERROR) - logger.info('+ JobAccessView/GET Success for %s/"%s"', jr_id, user.username) + logger.info('+ JobAccessView/GET Success for %s/"%s" on %s', + jr_id, user.username, jr.squonk_project) return Response(ok_response) From b382dfe909fabfc4d947ab6b9f2b35f12613e39a Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 7 Mar 2023 16:46:51 +0000 Subject: [PATCH 084/112] Fixed job request ID handling --- viewer/views.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/viewer/views.py b/viewer/views.py index 9a981cd1..ae6858e7 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3621,7 +3621,7 @@ class JobAccessView(APIView): - job_request_id: The identity of the JobRequest (the Job) the user needs access to Returns: A structure with 'accessible' set to True on success. On failure - an 'error' property contains a reason (string) + an 'error' string contains a reason example input for get @@ -3648,13 +3648,17 @@ def get(self, request): # Can't use this method if the squonk variables are not set! sqa_rv = _SQ2A.configured() if not sqa_rv.success: - err_response['error'] = f'Squonk is not configured for this stack' + err_response['error'] = f'Squonk is not available ({sqa_rv.msg})' return Response(err_response, status=status.HTTP_403_FORBIDDEN) - # Get the JobRequest Record - jr_id = request.query_params.get('job_request_id', None) - if not jr_id or jr_id < 1: - err_response['error'] = f'The JobRequest ID ("{jr_id}") is invalid' + # Get the JobRequest Record form the supplied record ID + jr_id_str = request.query_params.get('job_request_id', None) + if not jr_id_str or not jr_id_str.isdigit(): + err_response['error'] = f'The JobRequest ID ({jr_id_str}) is not valid' + return Response(err_response, status=status.HTTP_400_BAD_REQUEST) + jr_id = int(jr_id_str) + if jr_id < 1: + err_response['error'] = f'The JobRequest ID ({jr_id}) cannot be less than 1' return Response(err_response, status=status.HTTP_400_BAD_REQUEST) jr_list = JobRequest.objects.filter(id=jr_id) @@ -3665,7 +3669,7 @@ def get(self, request): # JobRequest must have a Squonk Project value if not jr.squonk_project: - err_response['error'] = f'The JobRequest has no Squonk Project value ({jr_id})' + err_response['error'] = f'The JobRequest ({jr_id}) has no Squonk Project value' return Response(err_response, status=status.HTTP_403_FORBIDDEN) # User must have access to the Job's Project. From db4ac24140effc3a94cfa8b2c96c576c68f01d5c Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 9 Mar 2023 09:24:16 +0000 Subject: [PATCH 085/112] Fixed access-id (now correctly using int) --- viewer/views.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/viewer/views.py b/viewer/views.py index ae6858e7..28909510 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3686,9 +3686,10 @@ def get(self, request): ' for JobRequest %s - granting access', jr_id) else: - # The project title is the Job access string. + # The project is the Job's access ID. # To check access we need this and the User's ID - access_id = jr.project.title + access_id = jr.project.id + access_string = jr.project.title sq2a_common_params: CommonParams = CommonParams(user_id=user.id, access_id=access_id, session_id=None, @@ -3698,7 +3699,7 @@ def get(self, request): callback_url=None) sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params) if not sq2a_rv.success: - err_response['error'] = f'Access to the Job for {access_id} is denied. {sq2a_rv.msg}' + err_response['error'] = f'Access to the Job for {access_id} ({access_string}) is denied. {sq2a_rv.msg}' return Response(err_response, status=status.HTTP_403_FORBIDDEN) # All checks passed ... grant access for this user From 3c15a368b8f4c69dd3d0c8e7677c8f9b9e9a2185 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 9 Mar 2023 10:08:33 +0000 Subject: [PATCH 086/112] Fixed migration conflict --- ...onstraint.py => 0028_protein_target_id_unique_constraint.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename viewer/migrations/{0025_protein_target_id_unique_constraint.py => 0028_protein_target_id_unique_constraint.py} (82%) diff --git a/viewer/migrations/0025_protein_target_id_unique_constraint.py b/viewer/migrations/0028_protein_target_id_unique_constraint.py similarity index 82% rename from viewer/migrations/0025_protein_target_id_unique_constraint.py rename to viewer/migrations/0028_protein_target_id_unique_constraint.py index 1bffaa68..66be7589 100644 --- a/viewer/migrations/0025_protein_target_id_unique_constraint.py +++ b/viewer/migrations/0028_protein_target_id_unique_constraint.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('viewer', '0024_add_job_request_start_and_finish_times'), + ('viewer', '0027_sessionproject_project.py'), ] operations = [ From 1f933a37224c698366ba4f5aa48c228eda89f6c9 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 9 Mar 2023 10:14:35 +0000 Subject: [PATCH 087/112] Fixed typo in migration --- viewer/migrations/0028_protein_target_id_unique_constraint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viewer/migrations/0028_protein_target_id_unique_constraint.py b/viewer/migrations/0028_protein_target_id_unique_constraint.py index 66be7589..2595bd3a 100644 --- a/viewer/migrations/0028_protein_target_id_unique_constraint.py +++ b/viewer/migrations/0028_protein_target_id_unique_constraint.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('viewer', '0027_sessionproject_project.py'), + ('viewer', '0027_sessionproject_project'), ] operations = [ From 1ed2f9ad4fd4268e9e9f262a7ca83fd4a822e11c Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 9 Mar 2023 12:12:17 +0000 Subject: [PATCH 088/112] Fixed merge conflict --- ...d_job_request_project.py => 0029_add_job_request_project.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename viewer/migrations/{0028_add_job_request_project.py => 0029_add_job_request_project.py} (87%) diff --git a/viewer/migrations/0028_add_job_request_project.py b/viewer/migrations/0029_add_job_request_project.py similarity index 87% rename from viewer/migrations/0028_add_job_request_project.py rename to viewer/migrations/0029_add_job_request_project.py index 6e05e615..d4a04b22 100644 --- a/viewer/migrations/0028_add_job_request_project.py +++ b/viewer/migrations/0029_add_job_request_project.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('viewer', '0027_sessionproject_project'), + ('viewer', '0028_protein_target_id_unique_constraint'), ] operations = [ From 93791d11e692a302acaa719fa97c861b69137412 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 10 Mar 2023 11:47:49 +0000 Subject: [PATCH 089/112] Initial logic to check existing units/products/projects --- requirements.txt | 2 +- viewer/squonk2_agent.py | 169 +++++++++++++++++++++++++++++++--------- 2 files changed, 132 insertions(+), 39 deletions(-) diff --git a/requirements.txt b/requirements.txt index 191f725c..9f631c42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.18.0 +im-squonk2-client==1.19.0 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 6a3a7ab0..26b3c0f0 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -26,7 +26,8 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) -# Response value for the agent methods +# Response value for the agent methods. +# It's a boolean and an optional string (used for errors or response content) Squonk2AgentRv: namedtuple = namedtuple('Squonk2AgentRv', ['success', 'msg']) SuccessRv: Squonk2AgentRv = Squonk2AgentRv(success=True, msg=None) @@ -334,6 +335,116 @@ def _pre_flight_checks(self) -> Squonk2AgentRv: # Organisation is known to AS, and it hasn't changed. return SuccessRv + def _get_or_create_unit(self, + unit_name_truncated: str, + unit_name_full: str) -> Squonk2AgentRv: + """Gets an exiting Unit or creates a new one + returning its UUID as the msg content. + """ + # Get existing Units for our Organisation + org_uuid: str = self.__org_record.uuid + as_rv: AsApiRv = AsApiRv.get_units(self.__org_owner_as_token, org_id=org_uuid) + if not as_rv.success: + msg: str = as_rv.msg['error'] + _LOGGER.error('Failed to get Units for Organisation "%s"', org_uuid) + return Squonk2AgentRv(success=False, msg=msg) + # Iterate through them, looking for ours... + _LOGGER.info('Got %s Units for Organisation "%s"', len(as_rv.msg['units']), org_uuid) + for unit in as_rv.msg['units']: + if unit['name'] == unit_name_truncated: + unit_uuid: str = unit['id'] + _LOGGER.info('...and one was ours (%s)', unit_uuid) + return Squonk2AgentRv(success=True, msg=unit_uuid) + + # Not found, create it... + _LOGGER.info('No exiting Unit, creating NEW Unit "%s" (for "%s")', + unit_name_full, + unit_name_truncated) + as_rv = AsApi.create_unit(self.__org_owner_as_token, + unit_name=unit_name_truncated, + org_id=org_uuid, + billing_day=self.__unit_billing_day) + if not as_rv.success: + msg = as_rv.msg['error'] + _LOGGER.error('Failed to create Unit "%s" (for "%s")', + unit_name_full, unit_name_truncated) + return Squonk2AgentRv(success=False, msg=msg) + + unit_uuid = as_rv.msg['id'] + _LOGGER.info('Created NEW Unit "%s"', unit_uuid) + return Squonk2AgentRv(success=True, msg=unit_uuid) + + def _get_or_create_product(self, name_truncated: str, unit_uuid: str) -> Squonk2AgentRv: + + # Get existing Products for the Unit + as_rv: AsApiRv = AsApiRv.get_products_for_unit(self.__org_owner_as_token, + unit_id=unit_uuid) + if not as_rv.success: + msg: str = as_rv.msg['error'] + _LOGGER.error('Failed to get Products for Unit "%s"', unit_uuid) + return Squonk2AgentRv(success=False, msg=msg) + + # Iterate through them, looking for ours... + for product in as_rv.msg['products']: + if product['product']['name'] == name_truncated: + product_uuid: str = product['product']['id'] + _LOGGER.info('Found pre-existing AS Product "%s"', product_uuid) + return Squonk2AgentRv(success=True, msg=product_uuid) + + # No existing Product with this name... + _LOGGER.info('No existing Product, creating NEW AS Product "%s" (for "%s")', + name_truncated, + unit_uuid) + as_rv = AsApi.create_product(self.__org_owner_as_token, + product_name=name_truncated, + unit_id=unit_uuid, + product_type=_SQ2_PRODUCT_TYPE, + flavour=self.__CFG_SQUONK2_PRODUCT_FLAVOUR) + if not as_rv.success: + msg = f'Failed to create AS Product ({as_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + product_uuid = as_rv.msg['id'] + msg = f'Created NEW AS Product {product_uuid}...' + _LOGGER.info(msg) + return Squonk2AgentRv(success=True, msg=product_uuid) + + def _get_or_create_project(self, name_truncated: str, product_uuid: str) -> Squonk2AgentRv: + """Gets existing DM Projects (that belong to the given Product) + to see if ours exists, if not a new one is created. + """ + # TODO + dm_rv: DmApiRv = DmApiRv.get_available_projects(self.__org_owner_as_token) + if not dm_rv.success: + msg: str = dm_rv.msg['error'] + _LOGGER.error('Failed to get Projects') + return Squonk2AgentRv(success=False, msg=msg) + + # Iterate through them, looking for ours... + for project in dm_rv.msg['projects']: + if project['name'] == name_truncated and 'product_id' in project and project['product_id'] == product_uuid: + project_uuid: str = project['project_id'] + _LOGGER.info('Found pre-existing DM Project "%s"', project_uuid) + return Squonk2AgentRv(success=True, msg=project_uuid) + + # No existing Project + _LOGGER.info('No existing Project, creating NEW DM Project "%s" (for "%s")', + name_truncated, + product_uuid) + dm_rv = DmApi.create_project(self.__org_owner_dm_token, + project_name=name_truncated, + private=True, + as_tier_product_id=product_uuid) + if not dm_rv.success: + msg = f'Failed to create DM Project ({dm_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + project_uuid: str = dm_rv.msg['project_id'] + msg = f'Created NEW DM Project {project_uuid}...' + return Squonk2AgentRv(success=True, msg=project_uuid) + def _delete_as_product(self, product_uuid: str) -> None: """Used in error conditions to remove a previously created Product. If this fails there's nothing else we can do so we just return regardless. @@ -378,41 +489,32 @@ def _create_product_and_project(self, # Create an AS Product. name_truncated, _ = self._build_product_name(user_name, session_title) - msg: str = f'Creating NEW AS Product "{name_truncated}" (unit={unit.uuid})...' + msg: str = f'Creating AS Product "{name_truncated}" (unit={unit.uuid})...' _LOGGER.info(msg) - as_rv: AsApiRv = AsApi.create_product(self.__org_owner_as_token, - product_name=name_truncated, - unit_id=unit.uuid, - product_type=_SQ2_PRODUCT_TYPE, - flavour=self.__CFG_SQUONK2_PRODUCT_FLAVOUR) - if not as_rv.success: - msg = f'Failed to create AS Product ({as_rv.msg})' + sq2_rv: Squonk2AgentRv = self._get_or_create_product(name_truncated, unit.uuid) + if not sq2_rv.success: + msg = f'Failed to create AS Product ({sq2_rv.msg})' _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - - product_uuid: str = as_rv.msg['id'] - msg = f'Created AS Product {product_uuid}...' + product_uuid: str = sq2_rv.msg + msg = f'Got or created AS Product {product_uuid}' _LOGGER.info(msg) # Create a DM Project (using the same name we used for the AS Product) - msg = f'Continuing by creating NEW DM Project "{name_truncated}"...' + msg = f'Continuing by creating DM Project "{name_truncated}"...' _LOGGER.info(msg) - dm_rv: DmApiRv = DmApi.create_project(self.__org_owner_dm_token, - project_name=name_truncated, - private=True, - as_tier_product_id=product_uuid) - if not dm_rv.success: - msg = f'Failed to create DM Project ({dm_rv.msg})' + sq2_rv: Squonk2AgentRv = self._get_or_create_project(name_truncated, product_uuid) + if not sq2_rv.success: + msg = f'Failed to create DM Project ({sq2_rv.msg})' _LOGGER.error(msg) # First delete the AS Product it should have been attached to self._delete_as_product(product_uuid) # Then leave... return Squonk2AgentRv(success=False, msg=msg) - - project_uuid: str = dm_rv.msg["project_id"] - msg = f'Created DM Project {project_uuid}...' + project_uuid: str = sq2_rv.msg + msg = f'Got or created DM Project {project_uuid}...' _LOGGER.info(msg) # Add the user as an Editor to the Project @@ -456,26 +558,17 @@ def _ensure_unit(self, target_access_string: int) -> Squonk2AgentRv: sq2_unit: Optional[Squonk2Unit] = Squonk2Unit.objects.filter(name=unit_name_full).first() if not sq2_unit: _LOGGER.info('No existing Squonk2Unit for "%s"', target_access_string) - rv: AsApiRv = AsApi.create_unit(self.__org_owner_as_token, - unit_name=unit_name_truncated, - org_id=self.__org_record.uuid, - billing_day=self.__unit_billing_day) - - _LOGGER.info('Creating NEW Squonk2Unit "%s" (for "%s")', - unit_name_full, - target_access_string) + # Get the list of Units from Squonk. + sq2a_rv: Squonk2AgentRv = self._get_or_create_unit(unit_name_truncated, unit_name_full) + if not sq2a_rv.success: + _LOGGER.error('Failed to create Unit "%s" (%s)', target_access_string, sq2a_rv.msg) + return Squonk2AgentRv(success=False, msg=sq2a_rv.msg) - if not rv.success: - msg: str = rv.msg['error'] - _LOGGER.error('Failed to create Unit "%s"', target_access_string) - return Squonk2AgentRv(success=False, msg=msg) - - unit_uuid: str = rv.msg['id'] + unit_uuid: str = sq2a_rv.msg sq2_unit = Squonk2Unit(uuid=unit_uuid, name=unit_name_full, organisation_id=self.__org_record.id) sq2_unit.save() - _LOGGER.info('Created Squonk2Unit %s "%s" (for "%s")', unit_uuid, unit_name_full, @@ -490,7 +583,7 @@ def _ensure_unit(self, target_access_string: int) -> Squonk2AgentRv: def _ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: """Gets or creates a Squonk2 Project, used as the destination of files - and job executions. Each project requires an AS Product + and job executions. Each Project requires an AS Product (tied to the User and Session) and Unit (tied to the Proposal/Project). The proposal is expected to be valid for a given user, this method does not From 101289e1db7f204edc4b9edaa1715c258d6887f3 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 10 Mar 2023 12:11:39 +0000 Subject: [PATCH 090/112] Fix API call typos --- viewer/squonk2_agent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 26b3c0f0..6cf7b69c 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -343,7 +343,7 @@ def _get_or_create_unit(self, """ # Get existing Units for our Organisation org_uuid: str = self.__org_record.uuid - as_rv: AsApiRv = AsApiRv.get_units(self.__org_owner_as_token, org_id=org_uuid) + as_rv: AsApiRv = AsApi.get_units(self.__org_owner_as_token, org_id=org_uuid) if not as_rv.success: msg: str = as_rv.msg['error'] _LOGGER.error('Failed to get Units for Organisation "%s"', org_uuid) @@ -377,8 +377,8 @@ def _get_or_create_unit(self, def _get_or_create_product(self, name_truncated: str, unit_uuid: str) -> Squonk2AgentRv: # Get existing Products for the Unit - as_rv: AsApiRv = AsApiRv.get_products_for_unit(self.__org_owner_as_token, - unit_id=unit_uuid) + as_rv: AsApiRv = AsApi.get_products_for_unit(self.__org_owner_as_token, + unit_id=unit_uuid) if not as_rv.success: msg: str = as_rv.msg['error'] _LOGGER.error('Failed to get Products for Unit "%s"', unit_uuid) @@ -415,7 +415,7 @@ def _get_or_create_project(self, name_truncated: str, product_uuid: str) -> Squo to see if ours exists, if not a new one is created. """ # TODO - dm_rv: DmApiRv = DmApiRv.get_available_projects(self.__org_owner_as_token) + dm_rv: DmApiRv = DmApi.get_available_projects(self.__org_owner_as_token) if not dm_rv.success: msg: str = dm_rv.msg['error'] _LOGGER.error('Failed to get Projects') From aca3874c49a38473a067f2c7316b3931e15bc864 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 17 Mar 2023 11:19:14 +0000 Subject: [PATCH 091/112] Fix for OIDC/Basic auth - Adds api/utils 'pretty_request()' - Switches UploadCSet/TSet to APIView (was View) --- api/utils.py | 33 +++++++++++++++++++++++++++++++++ fragalysis/settings.py | 7 ++++++- viewer/views.py | 32 ++++++++++++++++++++++++++------ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/api/utils.py b/api/utils.py index 4c9ca680..4d1c776c 100644 --- a/api/utils.py +++ b/api/utils.py @@ -321,3 +321,36 @@ def mol_view(request): return get_params(smiles, request) else: return HttpResponse("Please insert SMILES") + + +def pretty_request(request, *, tag=''): + """A simple function to return a Django request as nicely formatted string.""" + headers = '' + if request.headers: + for key, value in request.headers.items(): + headers += f'{key}: {value}\n' + + tag_text = f'{tag}\n' if tag else '' + + user_text = 'User: ' + if request.user: + user_text += str(request.user) + else: + user_text += '(-)' + + # Load the body but cater for problems, like + # django.http.request.RawPostDataException: + # You cannot access body after reading from request's data stream + body = None + try: + body = request.body + except Exception: + pass + + return f'{tag_text}' \ + '+ REQUEST BEGIN\n' \ + f'{user_text}\n' \ + f'{request.method} HTTP/1.1\n' \ + f'{headers}\n\n' \ + f'{body}\n' \ + '- REQUEST END' diff --git a/fragalysis/settings.py b/fragalysis/settings.py index c6652f99..3be8c542 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -76,6 +76,11 @@ "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", "PAGE_SIZE": 5000, "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.QueryParameterVersioning", + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + 'mozilla_django_oidc.contrib.drf.OIDCAuthentication', + 'rest_framework.authentication.BasicAuthentication', + ], } # CELERY STUFF @@ -357,7 +362,7 @@ # (50Mi of logging in 10 files of 5M each), # with the rotating file handler typically used for everything. DISABLE_LOGGING_FRAMEWORK = True if os.environ.get("DISABLE_LOGGING_FRAMEWORK", "no").lower() in ["yes"] else False -LOGGING_FRAMEWORK_ROOT_LEVEL = os.environ.get("LOGGING_FRAMEWORK_ROOT_LEVEL", "INFO") +LOGGING_FRAMEWORK_ROOT_LEVEL = os.environ.get("LOGGING_FRAMEWORK_ROOT_LEVEL", "DEBUG") if not DISABLE_LOGGING_FRAMEWORK: LOGGING = { 'version': 1, diff --git a/viewer/views.py b/viewer/views.py index 28909510..71bab54a 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -35,7 +35,7 @@ from api.security import ISpyBSafeQuerySet -from api.utils import get_params, get_highlighted_diffs +from api.utils import get_params, get_highlighted_diffs, pretty_request from viewer.utils import create_squonk_job_request_url from viewer.models import ( @@ -773,7 +773,7 @@ def save_tmp_file(myfile): return tmp_file -class UploadCSet(View): +class UploadCSet(APIView): """ View to render and control viewer/upload-cset.html - a page allowing upload of computed sets. Validation and upload tasks are defined in `viewer.compound_set_upload`, `viewer.sdf_check` and `viewer.tasks` and the task response handling is done by `viewer.views.ValidateTaskView` and `viewer.views.UploadTaskView` @@ -803,6 +803,11 @@ class UploadCSet(View): def get(self, request): + tag = '+ UploadCSet GET' + logger.info('%s', pretty_request(request, tag=tag)) + logger.info('User=%s', str(request.user)) +# logger.info('Auth=%s', str(request.auth)) + # Any messages passed to us via the session? # Maybe from a redirect? # It so take them and remove them. @@ -835,6 +840,11 @@ def get(self, request): def post(self, request): + tag = '+ UploadCSet POST' + logger.info('%s', pretty_request(request, tag=tag)) + logger.info('User=%s', str(request.user)) +# logger.info('Auth=%s', str(request.auth)) + # Only authenticated users can upload files # - this can be switched off in settings.py. user = self.request.user @@ -864,12 +874,12 @@ def post(self, request): # The 'sdf_file' anf 'target_name' are only required for upload/update sdf_file = request.FILES.get('sdf_file') target = request.POST.get('target_name') - update_set = request.POST['update_set'] + update_set = request.POST.get('update_set') # If a set is named the ComputedSet cannot be 'Anonymous' # and the user has to be the owner. selected_set = None - if update_set != 'None': + if update_set and update_set != 'None': computed_set_query = ComputedSet.objects.filter(unique_name=update_set) if computed_set_query: selected_set = computed_set_query[0] @@ -967,7 +977,7 @@ def post(self, request): # Upload Target datasets functions -class UploadTSet(View): +class UploadTSet(APIView): """ View to render and control viewer/upload-tset.html - a page allowing upload of computed sets. Validation and upload tasks are defined in `viewer.target_set_upload`, `viewer.sdf_check` and `viewer.tasks` and the task response handling is done by `viewer.views.ValidateTaskView` and `viewer.views.UploadTaskView` @@ -996,6 +1006,11 @@ class UploadTSet(View): def get(self, request): + tag = '+ UploadTSet GET' + logger.info('%s', pretty_request(request, tag=tag)) + logger.info('User="%s"', str(request.user)) +# logger.info('Auth="%s"', str(request.auth)) + # Only authenticated users can upload files - this can be switched off in settings.py. user = self.request.user if not user.is_authenticated and settings.AUTHENTICATE_UPLOAD: @@ -1013,7 +1028,12 @@ def get(self, request): return render(request, 'viewer/upload-tset.html', {'form': form}) def post(self, request): - logger.info('+ UploadTSet.post') + + tag = '+ UploadTSet POST' + logger.info('%s', pretty_request(request, tag=tag)) + logger.info('User="%s"', str(request.user)) +# logger.info('Auth="%s"', str(request.auth)) + context = {} # Only authenticated users can upload files - this can be switched off in settings.py. From a648780457c0ef0fca8de46d17cb66ab122b03bd Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 17 Mar 2023 11:20:20 +0000 Subject: [PATCH 092/112] Fixes exceptions in check_compound_set() - Fixes exceptions when 'generation_date' is missing or malformed --- viewer/sdf_check.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/viewer/sdf_check.py b/viewer/sdf_check.py index 95d489aa..2ea5c58d 100644 --- a/viewer/sdf_check.py +++ b/viewer/sdf_check.py @@ -18,7 +18,22 @@ def check_property_descriptions(): pass def check_compound_set(description_mol, validate_dict, update=None): - y_m_d = description_mol.GetProp('generation_date').split('-') + # Must have a 'generation_date' + if not description_mol.HasProp('generation_date'): + validate_dict = add_warning(molecule_name='File error', + field='compound set', + warning_string="Molecule has no generation_date", + validate_dict=validate_dict) + return validate_dict + # That's of the form "--"... + g_date = description_mol.GetProp('generation_date') + y_m_d = g_date.split('-') + if len(y_m_d) != 3: + validate_dict = add_warning(molecule_name='File error', + field='compound set', + warning_string=f"Molecule has no generation_date is not Y-M-D (g_date)", + validate_dict=validate_dict) + return validate_dict submitter_dict = {'submitter__name': description_mol.GetProp('submitter_name'), 'submitter__email': description_mol.GetProp('submitter_email'), From 5811b3da5127a30244aa50ef729126b1b700c993 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 23 Mar 2023 17:55:47 +0000 Subject: [PATCH 093/112] Initial work on job status refresh --- api/urls.py | 3 +- requirements.txt | 2 +- viewer/models.py | 2 + viewer/squonk2_agent.py | 49 ++++++++++++++++++ viewer/views.py | 112 +++++++++++++++++++++++++++------------- 5 files changed, 131 insertions(+), 37 deletions(-) diff --git a/api/urls.py b/api/urls.py index 8ce84069..daa8886f 100644 --- a/api/urls.py +++ b/api/urls.py @@ -85,7 +85,6 @@ # Squonk Jobs router.register(r"job_file_transfer", viewer_views.JobFileTransferView, basename='job_file_transfer') -router.register(r"job_request", viewer_views.JobRequestView, basename='job_request') router.register(r"job_callback", viewer_views.JobCallBackView, basename='job_callback') router.register(r"job_config", viewer_views.JobConfigView, basename='job_config') @@ -106,4 +105,6 @@ def schema_view(request): url(r"^", include(router.urls)), url(r"^auth$", drf_views.obtain_auth_token, name="auth"), url(r"^swagger$", schema_view), + + url(r"job_request", viewer_views.JobRequestView.as_view(), name="job_request"), ] diff --git a/requirements.txt b/requirements.txt index 9f631c42..33db6ac8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.19.0 +im-squonk2-client==1.20.0 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 diff --git a/viewer/models.py b/viewer/models.py index 09fd8dc8..9d3c63cb 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -1209,6 +1209,8 @@ class JobRequest(models.Model): class Meta: db_table = 'viewer_jobrequest' + def job_has_finished(self): + return self.job_status in [JobRequest.SUCCESS, JobRequest.FAILURE] class Squonk2Org(models.Model): """Django model to store Squonk2 Organisations (UUIDs) and the Account Servers diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 6cf7b69c..dd6e5949 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -947,6 +947,55 @@ def grant_access(self, a_params: AccessParams) -> Squonk2AgentRv: return SuccessRv + @synchronized + def get_instance_execution_status(self, instance_id: str) -> Squonk2AgentRv: + """A blocking method that attempt to get the execution status (success/failure) + of a given instance. The status (string) is returned as the Squonk2AgentRv.msg + value. + """ + assert instance_id + assert instance_id.startswith('instance-') + + if _TEST_MODE: + msg: str = 'Squonk2Agent is in TEST mode' + _LOGGER.warning(msg) + + # Every public API **MUST** call ping(). + # This ensures Squonk2 is available and gets suitable API tokens... + if not self.ping(): + msg = 'Squonk2 ping failed.'\ + ' Are we configured properly and is Squonk2 alive?' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # To do this we actually get the DM tasks + # (we can't currently filter on instance ID) + # And then look for the first one that is about our instance. + dm_rv: DmApiRv = DmApi.get_tasks(self.__org_owner_dm_token, + exclude_purpose='FILE.DATASET') + if not dm_rv.success: + msg = f'Failed to get DM Tasks ({dm_rv.msg})' + _LOGGER.error(msg) + return Squonk2AgentRv(success=False, msg=msg) + + # Search tasks for one relating to our instance... + # We return 'LOST' if a task cannot be found for th instance, + # otherwise it's one of None, 'SUCCESS or FAILURE + i_status: Optional[str] = 'LOST' + for i_task in dm_rv.msg['tasks']: + if i_task['purpose_id'] == instance_id: + # We've found a task for the instance... + if i_task['done']: + i_status = 'FAILURE' if i_task['exit_code'] != 0 else 'SUCCESS' + break + else: + i_status = None + if i_status and i_status == 'LOST': + msg = f'No Tasks found for {instance_id}' + _LOGGER.warning(msg) + + return Squonk2AgentRv(success=True, msg=i_status) + # A placeholder for the Agent object _AGENT_SINGLETON: Optional[Squonk2Agent] = None diff --git a/viewer/views.py b/viewer/views.py index 71bab54a..355271d0 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -24,12 +24,12 @@ from django.http import JsonResponse from django.views import View -from rest_framework import viewsets -from rest_framework.parsers import BaseParser +from rest_framework import status, viewsets from rest_framework.exceptions import ParseError -from rest_framework.views import APIView +from rest_framework.parsers import BaseParser +from rest_framework.renderers import JSONRenderer from rest_framework.response import Response -from rest_framework import status +from rest_framework.views import APIView from celery.result import AsyncResult @@ -3316,7 +3316,7 @@ def list(self, request): return Response(content) -class JobRequestView(viewsets.ModelViewSet): +class JobRequestView(APIView): """ Operational Django view to set up/retrieve information about tags relating to Session Projects @@ -3324,15 +3324,6 @@ class JobRequestView(viewsets.ModelViewSet): ------- url: api/job_request - queryset: - `viewer.models.JobRequest.objects.filter()` - filter fields: - - `viewer.models.JobRequest.snapshot` - ?snapshot= - - `viewer.models.JobRequest.target` - ?target= - - `viewer.models.JobRequest.user` - ?user= - - `viewer.models.JobRequest.squonk_job_name` - ?squonk_job_name= - - `viewer.models.JobRequest.squonk_project` - ?squonk_project= - - `viewer.models.JobRequest.job_status` - ?job_status= returns: JSON @@ -3359,30 +3350,81 @@ class JobRequestView(viewsets.ModelViewSet): """ - queryset = JobRequest.objects.filter() - filter_permissions = "target__project_id" - filterset_fields = ('id', 'snapshot', 'target', 'user', 'squonk_job_name', - 'squonk_project', 'job_status') + def get(self, request): + logger.info('+ JobRequest.get') + + user = self.request.user + if not user.is_authenticated: + content = {'Only authenticated users can access squonk jobs'} + return Response(content, status=status.HTTP_403_FORBIDDEN) - def get_serializer_class(self): - """Determine which serializer to use based on whether the request is a GET or a POST, PUT - or PATCH request + # Can't use this method if the Squonk2 agent is not configured + sq2a_rv = _SQ2A.configured() + if not sq2a_rv.success: + content = {f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'} + return Response(content, status=status.HTTP_403_FORBIDDEN) - Returns - ------- - Serializer (rest_framework.serializers.ModelSerializer): - - if GET: `viewer.serializers.JobRequestReadSerializer` - - if other: `viewer.serializers.JobRequestWriteSerializer - """ - if self.request.method in ['GET']: - # GET - return JobRequestReadSerializer - # (POST, PUT, PATCH) - return JobRequestWriteSerializer + # Iterate through each record, for JobRequests that are not 'finished' + # we call into Squonk to get an update. We then return the (possibly) updated + # records to the caller. - def create(self, request): - """Method to handle POST request - """ + results = [] + snapshot_id = request.query_params.get('snapshot_id', None) + + if snapshot_id: + logger.info('+ JobRequest.get snapshot_id=%s', snapshot_id) + job_requests = JobRequest.objects.filter(snapshot=int(snapshot_id)) + else: + logger.info('+ JobRequest.get snapshot_id=(unset)') + job_requests = JobRequest.objects.all() + + for jr in job_requests: + if not jr.job_has_finished(): + logger.info('+ JobRequest.get (id=%s) has not finished (job_status=%s)', + jr.id, jr.job_status) + # Job's not finished, an opportunity to call into Squonk + # To get the current status. To do this we'll need + # the instance ID (from the record's 'squonk_url_ext'). + # The URL is essentially a path which should end + # '/instalce-[...]'. + if jr.squonk_url_ext: + i_uuid = jr.squonk_url_ext.split('/')[-1] + if i_uuid.startswith('instance-'): + logger.info('+ JobRequest.get (id=%s, i_uuid=%s) getting update from Squonk...', + jr.id, i_uuid) + sq2a_rv = _SQ2A.get_instance_execution_status(i_uuid) + # If the job's now finished, update the record. + # We'll get None, 'LOST', 'SUCCESS' or 'FAILURE' + if not sq2a_rv.success: + logger.warning('+ JobRequest.get (id=%s, i_uuid=%s) check failed (%s)', + jr.id, i_uuid, sq2a_rv.msg) + elif sq2a_rv.success and sq2a_rv.msg: + logger.info('+ JobRequest.get (id=%s, i_uuid=%s) new status is (%s)', + jr.id, i_uuid, sq2a_rv.msg) + transition_time = str(datetime.utcnow()) + transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC) + jr.job_status = sq2a_rv.msg + jr.job_status_datetime = transition_time_utc + jr.job_finish_datetime = transition_time_utc + jr.save() + else: + logger.info('+ JobRequest.get (id=%s, i_uuid=%s) is (probably) still running', + jr.id, i_uuid) + + serializer = JobRequestReadSerializer(jr) + results.append(serializer.data) + + num_results = len(results) + logger.info('+ JobRequest.get num_results=%s', num_results) + + # Simulate the original paged API response... + content = {'count': num_results, + 'next': None, + 'previous': None, + 'results': results} + return Response(content, status=status.HTTP_200_OK) + + def post(self, request): # Celery/Redis must be running. # This call checks and trys to start them if they're not. assert check_services() From a9a7780792275c3a282d09f7ad7ee655fd79f7b4 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 23 Mar 2023 18:01:28 +0000 Subject: [PATCH 094/112] LOST now considered as finished --- viewer/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/viewer/models.py b/viewer/models.py index 9d3c63cb..7588aa6e 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -1210,7 +1210,9 @@ class Meta: db_table = 'viewer_jobrequest' def job_has_finished(self): - return self.job_status in [JobRequest.SUCCESS, JobRequest.FAILURE] + """Finished if status is SUCCESS or FAILURE (or a new state of LOST). + """ + return self.job_status in [JobRequest.SUCCESS, JobRequest.FAILURE, 'LOST'] class Squonk2Org(models.Model): """Django model to store Squonk2 Organisations (UUIDs) and the Account Servers From 74dcb8350497b3176901a1f7024f86b59e8f55b6 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 24 Mar 2023 11:34:38 +0000 Subject: [PATCH 095/112] Fixes snapshot (was snapshot_id) --- viewer/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viewer/views.py b/viewer/views.py index 355271d0..33088474 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3369,7 +3369,7 @@ def get(self, request): # records to the caller. results = [] - snapshot_id = request.query_params.get('snapshot_id', None) + snapshot_id = request.query_params.get('snapshot', None) if snapshot_id: logger.info('+ JobRequest.get snapshot_id=%s', snapshot_id) From de1287e459ce3331c8a40e09d38d5f8172324e07 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 24 Mar 2023 12:49:37 +0000 Subject: [PATCH 096/112] Now tries to get Squonk instance from command --- viewer/views.py | 66 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/viewer/views.py b/viewer/views.py index 33088474..67878b0e 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -2,6 +2,7 @@ import os import zipfile from io import StringIO +import re import uuid import shlex import shutil @@ -3382,35 +3383,56 @@ def get(self, request): if not jr.job_has_finished(): logger.info('+ JobRequest.get (id=%s) has not finished (job_status=%s)', jr.id, jr.job_status) + # Job's not finished, an opportunity to call into Squonk # To get the current status. To do this we'll need - # the instance ID (from the record's 'squonk_url_ext'). + # the instance ID. We can either get this from the record's + # 'squonk_url_ext' (set in the first callback). # The URL is essentially a path which should end - # '/instalce-[...]'. + # '/instalce-[...]'. If this fails, as a fall-back we also + # check the squonk_job_info, which may contain the instance in the command + i_uuid = None if jr.squonk_url_ext: - i_uuid = jr.squonk_url_ext.split('/')[-1] - if i_uuid.startswith('instance-'): - logger.info('+ JobRequest.get (id=%s, i_uuid=%s) getting update from Squonk...', + possible_uuid = jr.squonk_url_ext.split('/')[-1] + if possible_uuid.startswith('instance-'): + i_uuid = possible_uuid + logger.info('+ JobRequest.get (id=%s) found instance in squonk_url_ext (%s)', jr.id, i_uuid) - sq2a_rv = _SQ2A.get_instance_execution_status(i_uuid) - # If the job's now finished, update the record. - # We'll get None, 'LOST', 'SUCCESS' or 'FAILURE' - if not sq2a_rv.success: - logger.warning('+ JobRequest.get (id=%s, i_uuid=%s) check failed (%s)', - jr.id, i_uuid, sq2a_rv.msg) - elif sq2a_rv.success and sq2a_rv.msg: - logger.info('+ JobRequest.get (id=%s, i_uuid=%s) new status is (%s)', - jr.id, i_uuid, sq2a_rv.msg) - transition_time = str(datetime.utcnow()) - transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC) - jr.job_status = sq2a_rv.msg - jr.job_status_datetime = transition_time_utc - jr.job_finish_datetime = transition_time_utc - jr.save() - else: - logger.info('+ JobRequest.get (id=%s, i_uuid=%s) is (probably) still running', + if i_uuid is None and jr.squonk_job_info is not None: + # Not found it using the 'squonk_url_ext'. + # Let's try and find it in the command string.... + if 'command' in jr.squonk_job_info[1]: + cmd = jr.squonk_job_info[1]['command'] + match = re.search('(instance-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', cmd) + if match: + i_uuid = match.group(1) + logger.info('+ JobRequest.get (id=%s) found instance in squonk_job_info command (%s)', jr.id, i_uuid) + if i_uuid is None: + logger.info('+ JobRequest.get (id=%s) could not determine Instance', jr.id) + else: + logger.info('+ JobRequest.get (id=%s, i_uuid=%s) getting update from Squonk...', + jr.id, i_uuid) + sq2a_rv = _SQ2A.get_instance_execution_status(i_uuid) + # If the job's now finished, update the record. + # We'll get None, 'LOST', 'SUCCESS' or 'FAILURE' + if not sq2a_rv.success: + logger.warning('+ JobRequest.get (id=%s, i_uuid=%s) check failed (%s)', + jr.id, i_uuid, sq2a_rv.msg) + elif sq2a_rv.success and sq2a_rv.msg: + logger.info('+ JobRequest.get (id=%s, i_uuid=%s) new status is (%s)', + jr.id, i_uuid, sq2a_rv.msg) + transition_time = str(datetime.utcnow()) + transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC) + jr.job_status = sq2a_rv.msg + jr.job_status_datetime = transition_time_utc + jr.job_finish_datetime = transition_time_utc + jr.save() + else: + logger.info('+ JobRequest.get (id=%s, i_uuid=%s) is (probably) still running', + jr.id, i_uuid) + serializer = JobRequestReadSerializer(jr) results.append(serializer.data) From a4b69e9d2b3c6e5c24718fadb0f7055fe91f5f50 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 4 Apr 2023 11:16:36 +0200 Subject: [PATCH 097/112] Staging build now uses f/e staging branch --- .github/workflows/build-staging.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-staging.yaml b/.github/workflows/build-staging.yaml index 16c1eddf..c5e0197c 100644 --- a/.github/workflows/build-staging.yaml +++ b/.github/workflows/build-staging.yaml @@ -64,7 +64,7 @@ env: # For Jobs conditional on the presence of a secret see this Gist... # https://gist.github.com/jonico/24ffebee6d2fa2e679389fac8aef50a3 BE_NAMESPACE: xchem - FE_BRANCH: production + FE_BRANCH: staging FE_NAMESPACE: xchem STACK_BRANCH: master STACK_GITHUB_NAMESPACE: xchem From 9073d80c830062755a78fbf2818a71e731e95452 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 4 Apr 2023 16:28:07 +0200 Subject: [PATCH 098/112] Fix for /api/projects (indication of whether Jobs can br run) --- api/security.py | 2 +- viewer/serializers.py | 21 ++++++++++++++++++++- viewer/views.py | 3 ++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/api/security.py b/api/security.py index 6c399a3d..4bf2e072 100644 --- a/api/security.py +++ b/api/security.py @@ -132,7 +132,7 @@ def get_open_proposals(self): return ["lb00000", "OPEN", "private_dummy_project"] else: # A list of well-known (built-in) public Projects (Proposals/Visits) - return ["lb00000", "OPEN", "lb27156"] + return ["lb18453-1", "lb27156"] def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user diff --git a/viewer/serializers.py b/viewer/serializers.py index 4d467495..e18eac92 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -3,6 +3,7 @@ from frag.network.query import get_full_graph from urllib.parse import urljoin +from api.security import ISpyBSafeQuerySet from api.utils import draw_mol from viewer.models import ( @@ -38,6 +39,9 @@ from rest_framework import serializers +_ISPYB_SAFE_QUERY_SET = ISpyBSafeQuerySet() + + class FileSerializer(serializers.ModelSerializer): class Meta: model = File @@ -291,10 +295,20 @@ class ProjectSerializer(serializers.ModelSerializer): # 'authority' is the (as yet to be implemented) origin of the TAS # For now this is fixed at "DIAMOND-ISPYB" authority = serializers.SerializerMethodField() + # 'can_use_squonk' defines whether a user cna use Squonk for the Projetc + user_can_use_squonk = serializers.SerializerMethodField() def get_target_access_string(self, instance): return instance.title + def get_user_can_use_squonk(self, instance): + # User can use Squonk if there is a user object (i.e. they're authenticated) + # and ISPyB has the user in the Project + user = self.context['request'].user + if not user or instance.title not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(user): + return False + return True + def get_authority(self, instance): # Don't actually need the instance here. # We return a hard-coded string. @@ -303,7 +317,12 @@ def get_authority(self, instance): class Meta: model = Project - fields = ("id", "target_access_string", "init_date", "authority", "open_to_public") + fields = ("id", + "target_access_string", + "init_date", + "authority", + "open_to_public", + "user_can_use_squonk") class MolImageSerializer(serializers.ModelSerializer): diff --git a/viewer/views.py b/viewer/views.py index 67878b0e..12940674 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -432,7 +432,8 @@ class ProjectView(ISpyBSafeQuerySet): "target_access_string": "lb27156-1", "init_date": "2023-01-09T15:00:00Z", "authority": "DIAMOND-ISPYB", - "open_to_public": false + "open_to_public": false, + "user_can_use_squonk": false } ] From 7c70d8955a38f162f10a4b95ac9c55895d5bdf75 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 5 Apr 2023 11:55:08 +0200 Subject: [PATCH 099/112] Revert prior changes --- api/security.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/security.py b/api/security.py index 4bf2e072..6c399a3d 100644 --- a/api/security.py +++ b/api/security.py @@ -132,7 +132,7 @@ def get_open_proposals(self): return ["lb00000", "OPEN", "private_dummy_project"] else: # A list of well-known (built-in) public Projects (Proposals/Visits) - return ["lb18453-1", "lb27156"] + return ["lb00000", "OPEN", "lb27156"] def get_proposals_for_user_from_django(self, user): # Get the list of proposals for the user From 223522d734c5ae98349117d19df4b4c0d9d3eedd Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 6 Apr 2023 10:50:46 +0200 Subject: [PATCH 100/112] Better logging & LOST fix --- viewer/tasks.py | 24 +++++++++++---------- viewer/views.py | 55 +++++++++++++++++++++---------------------------- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/viewer/tasks.py b/viewer/tasks.py index 9920f23d..5141193e 100644 --- a/viewer/tasks.py +++ b/viewer/tasks.py @@ -479,7 +479,7 @@ def validate_target_set(target_zip, target=None, proposal=None, email=None): - submitter_name (str): name of the user who submitted the upload """ - logger.info('+ validating target set: ' + target_zip) + logger.info('+ TASK target_zip=%s', target_zip) # Get submitter name/info for passing into upload to get unique name submitter_name = '' @@ -559,7 +559,7 @@ def process_job_file_transfer(auth_token, id): """ - logger.info('+ Starting File Transfer (%s) [STARTED]', id) + logger.info('+ TASK Starting File Transfer (%s) [STARTED]', id) job_transfer = JobFileTransfer.objects.get(id=id) job_transfer.transfer_status = "STARTED" job_transfer.transfer_task_id = str(process_job_file_transfer.request.id) @@ -567,11 +567,11 @@ def process_job_file_transfer(auth_token, id): try: process_file_transfer(auth_token, job_transfer.id) except RuntimeError as error: - logger.error('- File Transfer failed %s', id) + logger.error('- TASK File transfer (%s) [FAILED] error=%s', + id, error) logger.error(error) job_transfer.transfer_status = "FAILURE" job_transfer.save() - logger.info('+ Failed File Transfer (%s) [FAILURE]', id) else: # Update the transfer datetime for comparison with the target upload datetime. # This should only be done on a successful upload. @@ -582,7 +582,7 @@ def process_job_file_transfer(auth_token, id): "compounds": list(SQUONK_COMP_MAPPING.keys())} job_transfer.transfer_spec = files_spec job_transfer.save() - logger.info('+ Successful File Transfer (%s) [SUCCESS]', id) + logger.info('+ TASK File transfer (%s) [SUCCESS]', id) return job_transfer.transfer_status @@ -611,7 +611,8 @@ def process_compound_set_job_file(task_params): job_output_path = task_params['job_output_path'] job_output_filename = task_params['job_output_filename'] - logger.info('+ Starting File Upload (%s)', jr_id) + logger.info('+ TASK task_params=%s', task_params) + job_request = JobRequest.objects.get(id=jr_id) job_request.upload_task_id = str(process_compound_set_job_file.request.id) job_request.save() @@ -623,12 +624,12 @@ def process_compound_set_job_file(task_params): job_output_path, job_output_filename) except RuntimeError as error: - logger.error('- File Upload failed (%s)', jr_id) + logger.error('- TASK file Upload failed (%s)', jr_id) logger.error(error) else: # Update the transfer datetime for comparison with the target upload datetime. # This should only be done on a successful upload. - logger.info('- File Upload Ended Successfully (%s)', jr_id) + logger.info('+ TASK file Upload Ended Successfully (%s)', jr_id) # We are expected to be followed by 'validate_compound_set' # which expects user_id, sdf_file and target @@ -660,6 +661,7 @@ def erase_compound_set_job_material(task_params, job_request_id=0): return job_request = JobRequest.objects.get(id=job_request_id) + logger.info('+ TASK Erasing job material job_request %s', job_request) # Upload done (successfully or not) # 'task_params' (a dictionary) is expected to be @@ -682,13 +684,13 @@ def erase_compound_set_job_material(task_params, job_request_id=0): job_request.computed_set = cs else: # Failed validation? - logger.info('Upload failed (%d) - task_params=%s', - job_request_id, task_params) + logger.info('- TASK Upload failed (%d) - task_params=%s', + job_request_id, task_params) logger.warning('Upload failed (%d) - process_stage value not satisfied', job_request_id) job_request.upload_status = 'FAILURE' job_request.save() - logger.info('Updated job_request %s', job_request) + logger.info('+ TASK Erased and updated job_request %s', job_request) # Always erase uploaded data delete_media_sub_directory(get_upload_sub_directory(job_request)) diff --git a/viewer/views.py b/viewer/views.py index 12940674..e83b327f 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3387,11 +3387,10 @@ def get(self, request): # Job's not finished, an opportunity to call into Squonk # To get the current status. To do this we'll need - # the instance ID. We can either get this from the record's - # 'squonk_url_ext' (set in the first callback). + # the instance ID, which, for the time-being we can only find in the + # callback URL - it is found in 'squonk_url_ext' (set in the first callback). # The URL is essentially a path which should end - # '/instalce-[...]'. If this fails, as a fall-back we also - # check the squonk_job_info, which may contain the instance in the command + # '/instalce-[...]'. i_uuid = None if jr.squonk_url_ext: possible_uuid = jr.squonk_url_ext.split('/')[-1] @@ -3399,17 +3398,6 @@ def get(self, request): i_uuid = possible_uuid logger.info('+ JobRequest.get (id=%s) found instance in squonk_url_ext (%s)', jr.id, i_uuid) - if i_uuid is None and jr.squonk_job_info is not None: - # Not found it using the 'squonk_url_ext'. - # Let's try and find it in the command string.... - if 'command' in jr.squonk_job_info[1]: - cmd = jr.squonk_job_info[1]['command'] - match = re.search('(instance-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', cmd) - if match: - i_uuid = match.group(1) - logger.info('+ JobRequest.get (id=%s) found instance in squonk_job_info command (%s)', - jr.id, i_uuid) - if i_uuid is None: logger.info('+ JobRequest.get (id=%s) could not determine Instance', jr.id) else: @@ -3577,8 +3565,7 @@ def update(self, request, code=None): break if not status_changed: - logger.info('code=%s status=%s ignoring (no status change)', - code, status) + logger.info('+ JobCallBackView.update(code=%s) status=%s ignoring (no status change)', code, status) return HttpResponse(status=204) # This is now a chance to safely set the squonk_url_ext using the instance ID @@ -3588,31 +3575,33 @@ def update(self, request, code=None): # into Fragalysis if not jr.squonk_url_ext: jr.squonk_url_ext = create_squonk_job_request_url(request.data['instance_id']) - logger.info("Setting jr.squonk_url_ext='%s'", jr.squonk_url_ext) + logger.info("+ JobCallBackView.update(code=%s) jr.squonk_url_ext='%s'", code, jr.squonk_url_ext) # Update the state transition time, # assuming UTC. transition_time = request.data.get('state_transition_time') if not transition_time: transition_time = str(datetime.utcnow()) - logger.warning("Callback is missing state_transition_time" - " (using '%s')", transition_time) + logger.warning("+ JobCallBackView.update(code=%s) callback is missing state_transition_time" + " (using '%s')", code, transition_time) transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC) jr.job_status_datetime = transition_time_utc - logger.info('code=%s status=%s transition_time=%s (new status)', + logger.info('+ JobCallBackView.update(code=%s) status=%s transition_time=%s (new status)', code, status, transition_time) # If the Job's start-time is not set, set it. if not jr.job_start_datetime: - logger.info('Setting job START datetime (%s)', transition_time) + logger.info('+ JobCallBackView.update(code=%s) setting job START datetime (%s)', + code, transition_time) jr.job_start_datetime = transition_time_utc # Set the Job's finish time (once) if it looks lie the Job's finished. # We can assume the Job's finished if the status is one of a number # of values... if not jr.job_finish_datetime and status in ('SUCCESS', 'FAILURE', 'REVOKED'): - logger.info('Setting job FINISH datetime (%s)', transition_time) + logger.info('+ JobCallBackView.update(code=%s) Setting job FINISH datetime (%s)', + code, transition_time) jr.job_finish_datetime = transition_time_utc # Save the JobRequest record before going further. @@ -3622,7 +3611,8 @@ def update(self, request, code=None): # Go no further unless SUCCESS return HttpResponse(status=204) - logger.info('Job finished (SUCCESS). Can we upload the results..?') + logger.info('+ JobCallBackView.update(code=%s) job finished (SUCCESS).' + ' Can we upload the results?', code) # SUCCESS ... automatic upload? # @@ -3647,21 +3637,21 @@ def update(self, request, code=None): job_output_path = '/' + os.path.dirname(job_output) job_output_filename = os.path.basename(job_output) - logging.info('job_output_path="%s"', job_output_path) - logging.info('job_output_filename="%s"', job_output_filename) + logging.info('+ JobCallBackView.update(code=%s) job_output_path="%s"', code, job_output_path) + logging.info('+ JobCallBackView.update(code=%s) job_output_filename="%s"', code, job_output_filename) # If it's not suitably named, leave expected_squonk_filename = 'merged.sdf' if job_output_filename != expected_squonk_filename: # Incorrectly named file - nothing to get/upload. - logger.info('SUCCESS but not uploading.' + logger.info('+ JobCallBackView.update(code=%s) SUCCESS but not uploading.' ' Expected "%s" as job_output_filename.' - ' Found "%s"', expected_squonk_filename, job_output_filename) + ' Found "%s"', code, expected_squonk_filename, job_output_filename) return HttpResponse(status=204) if jr.upload_status != 'PENDING': - logger.warning('SUCCESS but ignoring.' - ' upload_status=%s (already uploading?)', jr.upload_status) + logger.warning('+ JobCallBackView.update(code=%s) SUCCESS but ignoring.' + ' upload_status=%s (already uploading?)', code, jr.upload_status) return HttpResponse(status=204) # Change of status and SUCCESS @@ -3684,8 +3674,9 @@ def update(self, request, code=None): erase_compound_set_job_material.s(job_request_id=jr.id) ).apply_async() - logger.info('Started process_job_file_upload(%s) task_upload=%s', - jr.id, task_upload) + logger.info('+ JobCallBackView.update(code=%s)' + ' started process_job_file_upload(%s) task_upload=%s', + code, jr.id, task_upload) return HttpResponse(status=204) From 3836f67c42d456027e138119737b6b33932754ca Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Sun, 9 Apr 2023 08:42:56 +0200 Subject: [PATCH 101/112] - JobRequest 'code' now used as Squonk Job callback context - Refresh now searches using callback context - Now using im-squonk2-client 1.22.0 (callback context search) --- requirements.txt | 2 +- viewer/squonk2_agent.py | 46 ++++++++++++++++-------------- viewer/squonk_job_request.py | 40 ++++++++++++++------------ viewer/views.py | 54 ++++++++++++++---------------------- 4 files changed, 70 insertions(+), 72 deletions(-) diff --git a/requirements.txt b/requirements.txt index 33db6ac8..ac80a02e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ gunicorn==20.0.4 idna==2.10 image==1.5.32 importlib-metadata==1.7.0 -im-squonk2-client==1.20.0 +im-squonk2-client==1.22.0 ipaddress==1.0.23 ipython==7.17.0 ipython-genutils==0.2.0 diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index dd6e5949..9fe519f4 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -948,13 +948,12 @@ def grant_access(self, a_params: AccessParams) -> Squonk2AgentRv: return SuccessRv @synchronized - def get_instance_execution_status(self, instance_id: str) -> Squonk2AgentRv: + def get_instance_execution_status(self, callback_context: str) -> Squonk2AgentRv: """A blocking method that attempt to get the execution status (success/failure) - of a given instance. The status (string) is returned as the Squonk2AgentRv.msg - value. + of an instance (a Job) based on the given callback context. The status (string) + is returned as the Squonk2AgentRv.msg value. """ - assert instance_id - assert instance_id.startswith('instance-') + assert callback_context if _TEST_MODE: msg: str = 'Squonk2Agent is in TEST mode' @@ -968,30 +967,35 @@ def get_instance_execution_status(self, instance_id: str) -> Squonk2AgentRv: _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - # To do this we actually get the DM tasks - # (we can't currently filter on instance ID) - # And then look for the first one that is about our instance. + # To do this we actually get the DM tasks associated with the + # callback context that Fragalysis provided. + # For the filters we provide we should only get one Task. dm_rv: DmApiRv = DmApi.get_tasks(self.__org_owner_dm_token, - exclude_purpose='FILE.DATASET') + exclude_removal=True, + exclude_purpose='FILE.DATASET.PROJECT', + instance_callback_context=callback_context) if not dm_rv.success: msg = f'Failed to get DM Tasks ({dm_rv.msg})' _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) - # Search tasks for one relating to our instance... - # We return 'LOST' if a task cannot be found for th instance, - # otherwise it's one of None, 'SUCCESS or FAILURE + # Find the task that relates to our instance. + # We return 'LOST' if a task cannot be found, + # otherwise it's one of None, SUCCESS or FAILURE i_status: Optional[str] = 'LOST' - for i_task in dm_rv.msg['tasks']: - if i_task['purpose_id'] == instance_id: - # We've found a task for the instance... - if i_task['done']: - i_status = 'FAILURE' if i_task['exit_code'] != 0 else 'SUCCESS' - break - else: - i_status = None + num_tasks: int = len(dm_rv.msg['tasks']) + if num_tasks == 1: + i_task = dm_rv.msg['tasks'][0] + if i_task['done']: + i_status = 'FAILURE' if i_task['exit_code'] != 0 else 'SUCCESS' + else: + i_status = None + else: + msg = f'More than one Task found ({num_tasks}) for callback context "{callback_context}"' + _LOGGER.warning(msg) + if i_status and i_status == 'LOST': - msg = f'No Tasks found for {instance_id}' + msg = f'No Task found for callback context "{callback_context}", assume "LOST"' _LOGGER.warning(msg) return Squonk2AgentRv(success=True, msg=i_status) diff --git a/viewer/squonk_job_request.py b/viewer/squonk_job_request.py index cb74c318..1e301b10 100644 --- a/viewer/squonk_job_request.py +++ b/viewer/squonk_job_request.py @@ -87,7 +87,7 @@ def create_squonk_job(request): a URL allowing the front end to link to the running job instance """ - logger.info('+ create_squonk_job') + logger.info('+ create_squonk_job()') auth_token = request.session['oidc_access_token'] logger.debug(auth_token) @@ -107,17 +107,17 @@ def create_squonk_job(request): job_transfers = JobFileTransfer.objects.filter(snapshot=snapshot_id) if not job_transfers: - logger.warning('No JobFileTransfer object for snapshot %s', snapshot_id) + logger.warning('+ create_squonk_job() - No JobFileTransfer object for snapshot %s', snapshot_id) raise ValueError(f'No JobFileTransfer object for snapshot {snapshot_id}.' ' Files must be transferred before a job can be requested.') job_transfer = JobFileTransfer.objects.filter(snapshot=snapshot_id).latest('id') if job_transfer.transfer_status != 'SUCCESS': - logger.warning('JobFileTransfer status is not SUCCESS (is %s)', + logger.warning('+ create_squonk_job() - JobFileTransfer status is not SUCCESS (is %s)', job_transfer.transfer_status) raise ValueError('Job Transfer not complete') - logger.info('+ Calling ensure_project() to get the Squonk2 Project...') + logger.info('+ create_squonk_job() calling ensure_project() to get the Squonk2 Project') # This requires a Squonk2 Project (created by the Squonk2Agent). # It may be an existing project, or it might be a new project. @@ -128,7 +128,7 @@ def create_squonk_job(request): session_id=session_project_id) sq2_rv = _SQ2A.ensure_project(common_params) if not sq2_rv.success: - msg = f'JobTransfer failed to get/create a Squonk2 Project' \ + msg = f'+ create_squonk_job() - JobTransfer failed to get/create a Squonk2 Project' \ f' for User "{user.username}", Access ID {access_id},' \ f' Target ID {target_id}, and SessionProject ID {session_project_id}.' \ f' Got "{sq2_rv.msg}".' \ @@ -140,7 +140,7 @@ def create_squonk_job(request): squonk2_project_uuid = sq2_rv.msg.uuid squonk2_unit_name = sq2_rv.msg.unit.name squonk2_unit_uuid = sq2_rv.msg.unit.uuid - logger.info('+ ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)', + logger.info('+ create_squonk_job() ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)', squonk2_project_uuid, squonk2_unit_name, squonk2_unit_uuid) job_request = JobRequest() @@ -170,24 +170,26 @@ def create_squonk_job(request): # It is required to be a shortuuid of 22 characters using the default character set callback_token = shortuuid.uuid() - logger.info('+ job_name=%s', job_name) - logger.info('+ callback_url=%s', callback_url) - logger.info('+ callback_token=%s', callback_token) + logger.info('+ create_squonk_job() job_name=%s', job_name) + logger.info('+ create_squonk_job(%s) callback_url=%s', job_name, callback_url) + logger.info('+ create_squonk_job(%s) callback_token=%s', job_name, callback_token) # Dry-run the Job execution (this provides us with the 'command', which is # placed in the JobRecord's squonk_job_info). - logger.info('+ Calling DmApi.dry_run_job_instance(%s)', job_name) + logger.info('+ create_squonk_job(%s) code=%s calling DmApi.dry_run_job_instance()', + job_name, job_request.code) result = DmApi.dry_run_job_instance(auth_token, project_id=job_request.squonk_project, name=job_name, callback_url=callback_url, callback_token=callback_token, + callback_context=job_request.code, specification=json.loads(squonk_job_spec)) logger.debug(result) if not result.success: - logger.warning('+ dry_run_job_instance(%s) result=%s', job_name, result) - logger.error('+ FAILED (configuration problem) (%s)', job_name) + logger.warning('+ create_squonk_job(%s) code=%s dry_run_job_instance() FAILED result=%s', + job_name, job_request.code, result) job_request.delete() raise ValueError(result.msg) @@ -199,26 +201,30 @@ def create_squonk_job(request): job_request.save() # Now start the job 'for real'... - logger.info('+ Calling DmApi.start_job_instance(%s)', job_name) + logger.info('+ create_squonk_job(%s) code=%s calling DmApi.start_job_instance()', + job_name, job_request.code) result = DmApi.start_job_instance(auth_token, project_id=job_request.squonk_project, name=job_name, callback_url=callback_url, callback_token=callback_token, + callback_context=job_request.code, specification=json.loads(squonk_job_spec), timeout_s=8) logger.debug(result) if not result.success: - logger.warning('+ start_job_instance(%s) result=%s', job_name, result) - logger.error('+ FAILED (job probably did not start) (%s)', job_name) + logger.warning('+ create_squonk_job(%s) code=%s result=%s', + job_name, job_request.code, result) + logger.error('+ start_job_instance(%s) code=%s FAILED (job probably did not start)', + job_name, job_request.code) job_request.delete() raise ValueError(result.msg) job_instance_id = result.msg['instance_id'] job_task_id = result.msg['task_id'] - logger.info('+ SUCCESS. Job "%s" started (job_instance_id=%s job_task_id=%s)', - job_name, job_instance_id, job_task_id) + logger.info('+ create_squonk_job(%s) code=%s SUCCESS (job_instance_id=%s job_task_id=%s)', + job_name, job_request.code, job_instance_id, job_task_id) # Manufacture the Squonk URL (actually set in the callback) # so the front-end can use it immediately (we cannot set the JobRequest diff --git a/viewer/views.py b/viewer/views.py index e83b327f..76762d40 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3387,40 +3387,28 @@ def get(self, request): # Job's not finished, an opportunity to call into Squonk # To get the current status. To do this we'll need - # the instance ID, which, for the time-being we can only find in the - # callback URL - it is found in 'squonk_url_ext' (set in the first callback). - # The URL is essentially a path which should end - # '/instalce-[...]'. - i_uuid = None - if jr.squonk_url_ext: - possible_uuid = jr.squonk_url_ext.split('/')[-1] - if possible_uuid.startswith('instance-'): - i_uuid = possible_uuid - logger.info('+ JobRequest.get (id=%s) found instance in squonk_url_ext (%s)', - jr.id, i_uuid) - if i_uuid is None: - logger.info('+ JobRequest.get (id=%s) could not determine Instance', jr.id) + # the 'callback context' we supplied when launching the Job. + logger.info('+ JobRequest.get (id=%s, code=%s) getting update from Squonk...', + jr.id, jr.code) + sq2a_rv = _SQ2A.get_instance_execution_status(jr.code) + # If the job's now finished, update the record. + # If the call was successful we'll get None (not finished), + # 'LOST', 'SUCCESS' or 'FAILURE' + if not sq2a_rv.success: + logger.warning('+ JobRequest.get (id=%s, code=%s) check failed (%s)', + jr.id, jr.code, sq2a_rv.msg) + elif sq2a_rv.success and sq2a_rv.msg: + logger.info('+ JobRequest.get (id=%s, code=%s) new status is (%s)', + jr.id, jr.code, sq2a_rv.msg) + transition_time = str(datetime.utcnow()) + transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC) + jr.job_status = sq2a_rv.msg + jr.job_status_datetime = transition_time_utc + jr.job_finish_datetime = transition_time_utc + jr.save() else: - logger.info('+ JobRequest.get (id=%s, i_uuid=%s) getting update from Squonk...', - jr.id, i_uuid) - sq2a_rv = _SQ2A.get_instance_execution_status(i_uuid) - # If the job's now finished, update the record. - # We'll get None, 'LOST', 'SUCCESS' or 'FAILURE' - if not sq2a_rv.success: - logger.warning('+ JobRequest.get (id=%s, i_uuid=%s) check failed (%s)', - jr.id, i_uuid, sq2a_rv.msg) - elif sq2a_rv.success and sq2a_rv.msg: - logger.info('+ JobRequest.get (id=%s, i_uuid=%s) new status is (%s)', - jr.id, i_uuid, sq2a_rv.msg) - transition_time = str(datetime.utcnow()) - transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC) - jr.job_status = sq2a_rv.msg - jr.job_status_datetime = transition_time_utc - jr.job_finish_datetime = transition_time_utc - jr.save() - else: - logger.info('+ JobRequest.get (id=%s, i_uuid=%s) is (probably) still running', - jr.id, i_uuid) + logger.info('+ JobRequest.get (id=%s, code=%s) is (probably) still running', + jr.id, jr.code) serializer = JobRequestReadSerializer(jr) results.append(serializer.data) From 7918fb0f57f81511f635315c065fc5849361659a Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 11 Apr 2023 10:43:22 +0200 Subject: [PATCH 102/112] Fix for _ensure_unit() assert --- viewer/squonk2_agent.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 9fe519f4..0d25a8df 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -551,7 +551,11 @@ def _ensure_unit(self, target_access_string: int) -> Squonk2AgentRv: On success the returned message is used to carry the Squonk2 project UUID. """ - assert self.__org_record + if not self.__org_record: + msg: str = 'The Squonk2Org record does not match' \ + ' the configured SQUONK2_ORG_UUID.' \ + ' You cannot change the SQUONK2_ORG_UUID once it has been used' + return Squonk2AgentRv(success=False, msg=msg) # Now we check and create a Squonk2Unit... unit_name_truncated, unit_name_full = self._build_unit_name(target_access_string) @@ -875,7 +879,7 @@ def send(self, s_params: SendParams) -> Squonk2AgentRv: rv_u: Squonk2AgentRv = self._ensure_project(s_params.common) if not rv_u.success: - msg = 'Failed to create corresponding Squonk2 Project' + msg = f'Failed to create corresponding Squonk2 Project (msg={rv_u.msg})' _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) @@ -911,7 +915,7 @@ def ensure_project(self, c_params: CommonParams) -> Squonk2AgentRv: rv_u: Squonk2AgentRv = self._ensure_project(c_params) if not rv_u.success: - msg = 'Failed to create corresponding Squonk2 Project' + msg = f'Failed to create corresponding Squonk2 Project (msg={rv_u.msg})' _LOGGER.error(msg) return Squonk2AgentRv(success=False, msg=msg) From ecf8e20228a9e756bf0a7e45dde705ebc5b44482 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Tue, 11 Apr 2023 11:03:45 +0200 Subject: [PATCH 103/112] process_compound_set_file no longer checks the instance ID --- viewer/squonk_job_file_upload.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/viewer/squonk_job_file_upload.py b/viewer/squonk_job_file_upload.py index 6432f21e..6858536e 100644 --- a/viewer/squonk_job_file_upload.py +++ b/viewer/squonk_job_file_upload.py @@ -149,17 +149,6 @@ def process_compound_set_file(jr_id, callback_token = jr_job_info_msg.get('callback_token') logger.info("Squonk API callback_token=%s", callback_token) - # The Squonk instance that ran the Job can be found - # at the end of the jr.squonk_url_ext field. - instance_id = '' - if jr.squonk_url_ext: - i_id = jr.squonk_url_ext.split('/')[-1] - if i_id.startswith('instance-'): - instance_id = i_id - logger.info("Squonk instance_id='%s'", instance_id) - else: - logger.warning("jr.squonk_url_ext does not contain an instance ID") - # Do we need to create the upload path? # This is used for this 'job' and is removed when the upload is complete # successful or otherwise. From 79c2c6349ca784a2ef8a075ce2c6aa4e1856376f Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Wed, 12 Apr 2023 16:02:19 +0200 Subject: [PATCH 104/112] pretty_request() does not print body --- api/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api/utils.py b/api/utils.py index 4d1c776c..6b338bd5 100644 --- a/api/utils.py +++ b/api/utils.py @@ -323,7 +323,7 @@ def mol_view(request): return HttpResponse("Please insert SMILES") -def pretty_request(request, *, tag=''): +def pretty_request(request, *, tag='', print_body=False): """A simple function to return a Django request as nicely formatted string.""" headers = '' if request.headers: @@ -342,10 +342,11 @@ def pretty_request(request, *, tag=''): # django.http.request.RawPostDataException: # You cannot access body after reading from request's data stream body = None - try: - body = request.body - except Exception: - pass + if print_body: + try: + body = request.body + except Exception: + pass return f'{tag_text}' \ '+ REQUEST BEGIN\n' \ From 71ea2762f817a74ceba34ad722b91e9d49e06f98 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 13 Apr 2023 16:06:59 +0200 Subject: [PATCH 105/112] Improved path logging in UploadCSet POST --- viewer/models.py | 21 +++++++++++++++------ viewer/views.py | 19 ++++++++++++++++++- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/viewer/models.py b/viewer/models.py index 7588aa6e..70d2e952 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -1056,14 +1056,14 @@ class JobFileTransfer(models.Model): A foreign key link to the relevant target the file transfer is part of (required) squonk_project: CharField The name of a project that has been created in Squonk that the files will be transferred to - projects: JSONField + proteins: JSONField List of proteins to be transferred compounds: JSONField - List of coumpounds to be transferred (not used yet) + List of compounds to be transferred (not used yet) transfer_spec: JSONField Identifies for each type (protein or compound), which file types were transferred over. transfer_task_id: CharField - Task id of transfer celery task Note that if a resynchronisation is + Task id of transfer celery task Note that if a re-synchronisation is required this will be re-used. transfer_status: CharField Identifies the status of the transfer. @@ -1121,6 +1121,8 @@ class JobRequest(models.Model): A foreign key link to the relevant snapshot the file transfer is part of (required) target: ForeignKey A foreign key link to the relevant target the file transfer is part of (required) + project: ForeignKey + The Fragalysis Project record ths JobRequest is associated with squonk_project: CharField The name of a project that has been created in Squonk that the files will be transferred to squonk_job_spec: JSONField @@ -1141,9 +1143,13 @@ class JobRequest(models.Model): squonk_job_info: JSONField Squonk job information returned from the initial Squonk POST instance API call squonk_url_ext: CharField - Squonk URL information to be added to the Host URL to link to a Squonk Job + Squonk URL information to be added to the Host URL to link to a Squonk Job. + This field is populated during a call to the JobCallbackView. code: UUIDField A UUID generated by Fragalysis and passed to Squonk as part of a callback URL. + This value is used to uniquely identify the HJob in Squonk and is passed back + by squonk to provide context in calls to the JobCallBackView. + context in subs upload_task_id: CharField Celery task ID for results upload task (optional). Set when the Job completes and an automated upload follows. @@ -1253,7 +1259,7 @@ class Squonk2Unit(models.Model): A Foreign Key to the Project (Proposal) the Unit belongs to, a record that contains the "target access string". organisation: ForeignKey - A Foreign Key to the Organisation the Unit belongs to. + The Organisation the Unit belongs to. """ uuid = models.TextField(max_length=41, null=False) name = models.TextField(null=False) @@ -1262,7 +1268,8 @@ class Squonk2Unit(models.Model): organisation = models.ForeignKey(Squonk2Org, null=False, on_delete=models.CASCADE) class Squonk2Project(models.Model): - """Django model to store Squonk2 Project (UUIDs). Managed by the Squonk2Agent class. + """Django model to store Squonk2 Project and Product (UUIDs). + Managed by the Squonk2Agent class. Parameters ---------- @@ -1276,6 +1283,8 @@ class Squonk2Project(models.Model): A Squonk2 Account Server (AS) Product UUID. A fixed length string consisting of 'product-' followed by a uuid4 value, e.g. 'product-54260047-183b-42e8-9658-385a1e1bd236' + unit: ForeignKey + The Squonk2 Unit the Product (and Project) belongs to. """ uuid = models.TextField(max_length=44, null=False) name = models.TextField(null=False) diff --git a/viewer/views.py b/viewer/views.py index 76762d40..6b08e765 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -862,7 +862,6 @@ def post(self, request): assert check_services() form = CSetForm(request.POST, request.FILES) - if form.is_valid(): # Get all the variables needed from the form. @@ -878,6 +877,9 @@ def post(self, request): target = request.POST.get('target_name') update_set = request.POST.get('update_set') + logger.info('+ UploadCSet POST target=%s', target) + logger.info('+ UploadCSet POST update_set="%s"', update_set) + # If a set is named the ComputedSet cannot be 'Anonymous' # and the user has to be the owner. selected_set = None @@ -888,6 +890,7 @@ def post(self, request): else: request.session[_SESSION_ERROR] = \ 'The set could not be found' + logger.warning('- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]) return redirect('upload_cset') # If validating or uploading we need a Target and SDF file. @@ -897,11 +900,13 @@ def post(self, request): request.session[_SESSION_ERROR] = \ 'To Validate or Upload' \ ' you must provide a Target and SDF file' + logger.warning('- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]) return redirect('upload_cset') elif choice in ['D']: if update_set == 'None': request.session[_SESSION_ERROR] = \ 'To Delete you must select an existing set' + logger.warning('- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]) return redirect('upload_cset') # If uploading (updating) or deleting @@ -917,6 +922,7 @@ def post(self, request): # Something wrong? # If so redirect... if _SESSION_ERROR in request.session: + logger.warning('- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]) return redirect('upload_cset') # Save uploaded sdf and zip to tmp storage @@ -940,6 +946,8 @@ def post(self, request): task_params['update'] = update_set task_validate = validate_compound_set.delay(task_params) + logger.info('+ UploadCSet POST "Validate" task underway') + # Update client side with task id and status context = {'validate_task_id': task_validate.id, 'validate_task_status': task_validate.status} @@ -960,6 +968,8 @@ def post(self, request): validate_compound_set.s(task_params) | process_compound_set.s()).apply_async() + logger.info('+ UploadCSet POST "Upload" task underway') + # Update client side with task id and status context = {'upload_task_id': task_upload.id, 'upload_task_status': task_upload.status} @@ -971,7 +981,14 @@ def post(self, request): request.session[_SESSION_MESSAGE] = \ f'Compound set "{selected_set.unique_name}" deleted' + + logger.info('+ UploadCSet POST "Delete" done') + return redirect('upload_cset') + else: + logger.warning('- UploadCSet POST form.is_valid() returned False') + + logger.warning('- UploadCSet POST (leaving)') context = {'form': form} return render(request, 'viewer/upload-cset.html', context) From 6aff920e59872ce1f15c4ba0a479faa5e3baa048 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 13 Apr 2023 16:19:50 +0200 Subject: [PATCH 106/112] Adds Squonk integration doc (minor teaks to others) --- docs/source/API/download.rst | 6 +- docs/source/computational_data/views.rst | 7 - docs/source/index.rst | 1 + docs/source/squonk/integration.rst | 158 +++++++++++++++++++++++ requirements.txt | 1 - 5 files changed, 161 insertions(+), 12 deletions(-) create mode 100644 docs/source/squonk/integration.rst diff --git a/docs/source/API/download.rst b/docs/source/API/download.rst index fe5cce14..01e4bbd2 100644 --- a/docs/source/API/download.rst +++ b/docs/source/API/download.rst @@ -10,10 +10,8 @@ Fragalysis provides functionality to flexibly download subsets of data as follow - **Subset of Computed Set Data** - Constructs a csv file for download based on a dictionary constructed in the react front end from the computed sets. - - Views ---------- - +Views +----- .. autoclass:: viewer.views.DownloadStructures :members: diff --git a/docs/source/computational_data/views.rst b/docs/source/computational_data/views.rst index cbee16ee..0cffbccf 100644 --- a/docs/source/computational_data/views.rst +++ b/docs/source/computational_data/views.rst @@ -19,9 +19,6 @@ a version of the standard DRF (see :ref:`RESTful API (Views) `) :code is used to generate the 2D image shown on each molecule in the RHS computed set tab using :code:`viewer.views.img_from_smiles`. The scores for each molecule are displayed on the respective molecule card. -- :code:`viewer.views.cset_key`: This view is used to generate a computed set upload key at :code:`/viewer/cset_key` - , and email it to the user to allow them to upload new computed sets at :code:`/viewer/upload_cset` - - :code:`viewer.views.UploadCSet`: This view is used to generate the form found at :code:`/viewer/upload_cset`, and to pass the values to the celery tasks controlled through :code:`viewer.views.ValidateTaskView` and :code:`viewer.views.UploadTaskView` that validate and process the uploaded data, saving it into the relevant models @@ -52,9 +49,6 @@ View details .. autoclass:: viewer.views.ComputedMolAndScoreView :members: -.. autoclass:: viewer.views.cset_key - :members: - .. autoclass:: viewer.views.UploadCSet :members: @@ -72,4 +66,3 @@ View details .. autoclass:: viewer.views.pset_download :members: - diff --git a/docs/source/index.rst b/docs/source/index.rst index 8dbbc939..91becaa3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -26,6 +26,7 @@ Contents API/media API/download vectors + squonk/integration Indices and tables diff --git a/docs/source/squonk/integration.rst b/docs/source/squonk/integration.rst new file mode 100644 index 00000000..23dd7315 --- /dev/null +++ b/docs/source/squonk/integration.rst @@ -0,0 +1,158 @@ +Squonk Integration +================== + +The `Squonk`_ **Data Manager** and associated services, developed by `Informatics Matters`_, +provides a novel, easy to use, web based, data centric workflow environment in which +scientists can execute scientific workflows using open source and commercial tools from +multiple sources such as RDKit, Chemistry Development Kit (CDK), ChemAxon. + +A RESTful API is also available for accessing the data and services **Squonk** provides. + +Fragalysis has been adapted so that it can utilise **Squonk** services via its API +and Graphical User Interface. + +Squonk Installation +------------------- + +Rather than use the commercial **Squonk** service a *custom* **Squonk** installation +is typically provided, running in the same cluster as the Fragalysis Stack that +you wish to use. The installation will consist of the **Squonk** *Data Manager* +it accounting service, the *account server*. The *Data Manager* and *Account Server* +will be configured to use the keycloak authentication server also used by Fragalysis. + +The administrator of these applications will need to ensure the following: - + +Fragalysis Squonk User +^^^^^^^^^^^^^^^^^^^^^^ + +An administrative user is made available for use exclusively by Fragalysis. This is +the user Fragalysis will use to automate the orchestration of objects in **Squonk** - +the creation of *Units*, *Products*, *Projects* and Jobs (*Instances*). The user name +is typically `fragalysis`. + +.. note:: + If more than one Fragalysis Stack is configured to use one **Squonk** installation + the user can be shared between them - you don't necessarily need a separate + `fragalysis` user for each Fragalysis Stack. + +Fragalysis Squonk Organisation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An *Organisation* needs to be created that will be the root of all objects created +by Fragalysis (its *Units*, *Products* and *Projects*). You don't need to know +how these objects relate to Fragalysis activity, only that an *Organisation* must +be provisioned. + +.. note:: + The Organisation is Stack-specific - **EVERY** Fragalysis Stack must be assigned + a unique organisation. The *Staging* and *Production* stacks currently share + access to One **Squonk**, and they each do this using separate *Organisations*. + +Squonk User Roles +^^^^^^^^^^^^^^^^^ + +Users that use Fragalysis, and also need access to **Squonk** services, +will need to be assigned the ``data-manager-user`` and ``account-server-user`` **Roles** +in the corresponding keycloak service. + +Squonk Integration +------------------ + +With the **Squonk** installation in place, the Fragalysis Stack needs be configured +using the following Ansible playbook variables, which translate to environment variables +available to the Fragalysis Stack container: - + +.. list-table:: Stack Playbook Variables + :widths: 30 70 + :header-rows: 1 + + * - Variable + - Description + * - ``stack_oidc_as_client_id`` + - The client ID of the Account Server in the keycloak authentication server. + * - ``stack_oidc_dm_client_id`` + - The client ID of the Data Manager in the keycloak authentication server. + * - ``stack_squonk2_ui_url`` + - The URL of the installed Squonk UI, typically ``https://squonk.example.com/data-manager-ui`` + * - ``stack_squonk2_dmapi_url`` + - The URL of the installed Squonk Data Manager API, typically ``https://squonk.example.com/data-manager-api`` + * - ``stack_squonk2_asapi_url`` + - The URL of the installed Squonk Account Server API, typically ``https://squonk.example.com/account-server-api`` + * - ``stack_squonk2_org_owner`` + - The name of the Fragalysis user that will be used to create objects in Squonk. + * - ``stack_squonk2_org_owner_password`` + - The password of the Fragalysis user that will be used to create objects in Squonk. + * - ``stack_squonk2_org_uuid`` + - The UUID of the Squonk Organisation under which *Units*, *Products* and *Projects* + will be created. This must be unique for each Fragalysis Stack. + * - ``stack_squonk2_product_flavour`` + - The *flavour* of the Squonk *Products* that will be created. + This must be one of ``BRONZE``, ``SILVER`` or ``GOLD``. + * - ``stack_squonk2_slug`` + - The *slug* used to create objects in Squonk *Product*. This is used to + identify the Fragalysis Stack that created the object and must be + unique for each Fragalysis Stack. It is a short string (up to 10 characters). + The staging stack might use ``staging`` and the production stack might use + ``production``. + * - ``stack_squonk2_unit_billing_day`` + - The day of the month on which the *Units* will be billed. This must be + an integer between 1 and 28. + +.. warning:: + The ``stack_squonk2_org_uuid``, which must be unique for each Fragalysis Stack, + cannot currently be changed once the stack has been launched. It is the + responsibility of the administrator to ensure that the UUID is unique and the + variable is correctly set prior to launching the stack. + +Squonk Model +------------ + +The following Fragalysis ``viewer.models`` are used by the Fragalysis Stack to manage +access to and record the **Squonk** objects that are created: - + +.. autoclass:: viewer.models.Squonk2Org + :members: + +.. autoclass:: viewer.models.Squonk2Unit + :members: + +.. autoclass:: viewer.models.Squonk2Project + :members: + +.. autoclass:: viewer.models.JobFileTransfer + :members: + +.. autoclass:: viewer.models.JobRequest + :members: + +Squonk Views +------------ + +.. autoclass:: viewer.views.JobConfigView + :members: + +.. autoclass:: viewer.views.JobFileTransferView + :members: + +.. autoclass:: viewer.views.JobRequestView + :members: + +.. autoclass:: viewer.views.JobCallBackView + :members: + +.. autoclass:: viewer.views.JobAccessView + :members: + +Squonk2Agent Class +------------------ + +The main interactions with **Squonk** are handled by the ``Squonk2Agent`` class +in the ``viewer`` package and the `Squonk2 Python Client`_, which provides API access +to **Squonk** through its ``DmApi`` and ``AsApi`` classes. + +.. autoclass:: viewer.squonk2_agent.Squonk2Agent + :members: + +.. _Informatics Matters: https://www.informaticsmatters.com/ +.. _Squonk: https://squonk.it/ +.. _Squonk2 Python Client: https://github.com/InformaticsMatters/squonk2-python-client diff --git a/requirements.txt b/requirements.txt index ac80a02e..731d7d55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -132,4 +132,3 @@ xchem-db==0.1.26b0 yarl==1.5.1 zipp==3.1.0 pymysql~=0.10.1 -sphinx From 5fee0876dd774c4a859315ec94b1077301626882 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Thu, 13 Apr 2023 16:20:15 +0200 Subject: [PATCH 107/112] Better cocker-compose (healthchecks) --- docker-compose.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c494c2c1..6ecbaeb5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,11 +21,16 @@ services: - ../data/postgresql/data:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: fragalysis - POSTGRES_USER: fragalysis POSTGRES_DB: frag PGDATA: /var/lib/postgresql/data/pgdata ports: - "5432:5432" + healthcheck: + test: pg_isready -U postgres -d frag + interval: 10s + timeout: 2s + retries: 5 + start_period: 10s # The graph graph: @@ -45,6 +50,12 @@ services: environment: - NEO4J_AUTH=none - NEO4J_dbms_memory_pagecache_size=4G + healthcheck: + test: wget http://localhost:7474 || exit 1 + interval: 10s + timeout: 10s + retries: 20 + start_period: 10s # The stack (backend) stack: @@ -60,7 +71,7 @@ services: - .:/code/ environment: POSTGRESQL_DATABASE: frag - POSTGRESQL_USER: fragalysis + POSTGRESQL_USER: postgres POSTGRESQL_PASSWORD: fragalysis POSTGRESQL_HOST: database POSTGRESQL_PORT: 5432 @@ -92,5 +103,7 @@ services: ports: - "8080:80" depends_on: - - database - - graph + database: + condition: service_healthy + graph: + condition: service_healthy From e30608fd2549506f16797eeccf760a1f21d882d2 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 17 Apr 2023 16:38:01 +0200 Subject: [PATCH 108/112] Improved logging on update CSet --- viewer/views.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/viewer/views.py b/viewer/views.py index 6b08e765..dac210c2 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -877,8 +877,7 @@ def post(self, request): target = request.POST.get('target_name') update_set = request.POST.get('update_set') - logger.info('+ UploadCSet POST target=%s', target) - logger.info('+ UploadCSet POST update_set="%s"', update_set) + logger.info('+ UploadCSet POST choice="%s" target="%s" update_set="%s"', choice, target, update_set) # If a set is named the ComputedSet cannot be 'Anonymous' # and the user has to be the owner. @@ -985,10 +984,14 @@ def post(self, request): logger.info('+ UploadCSet POST "Delete" done') return redirect('upload_cset') + + else: + logger.warning('+ UploadCSet POST unsupported submit_choice value (%s)', choice) + else: logger.warning('- UploadCSet POST form.is_valid() returned False') - logger.warning('- UploadCSet POST (leaving)') + logger.info('- UploadCSet POST (leaving)') context = {'form': form} return render(request, 'viewer/upload-cset.html', context) From f6e40e58e2520344a3eee9516e699831be7ee221 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 21 Apr 2023 16:00:18 +0200 Subject: [PATCH 109/112] Adds sub-path to Job transfer --- viewer/models.py | 3 +++ viewer/squonk_job_file_transfer.py | 2 +- viewer/views.py | 20 +++++++++----------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/viewer/models.py b/viewer/models.py index 70d2e952..4c41a867 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -6,6 +6,8 @@ from django.core.validators import MinLengthValidator from django.conf import settings +from shortuuid.django_fields import ShortUUIDField + from simple_history.models import HistoricalRecords from viewer.target_set_config import get_mol_choices, get_prot_choices @@ -1091,6 +1093,7 @@ class JobFileTransfer(models.Model): snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE) target = models.ForeignKey(Target, null=True, on_delete=models.CASCADE, db_index=True) squonk_project = models.CharField(max_length=200, null=True) + sub_path = ShortUUIDField(length=4, alphabet="abcdefghijklmnopqrstuvwxyz") proteins = models.JSONField(encoder=DjangoJSONEncoder, null=True) # Not used in phase 1 compounds = models.JSONField(encoder=DjangoJSONEncoder, null=True) diff --git a/viewer/squonk_job_file_transfer.py b/viewer/squonk_job_file_transfer.py index 30bf27c4..4625ccd1 100644 --- a/viewer/squonk_job_file_transfer.py +++ b/viewer/squonk_job_file_transfer.py @@ -250,7 +250,7 @@ def process_file_transfer(auth_token, # location in squonk project where files will reside # e.g. "/fragalysis-files/Mpro" target = job_transfer.target - squonk_directory = '/' + settings.SQUONK2_MEDIA_DIRECTORY + '/' + target.title + squonk_directory = os.path.join('/', settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path, target.title) logger.info('+ Destination squonk_directory=%s', squonk_directory) # This to pick up NULL values from the changeover to using compounds. diff --git a/viewer/views.py b/viewer/views.py index dac210c2..8592b842 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -1,3 +1,4 @@ +import logging import json import os import zipfile @@ -11,8 +12,7 @@ from dateutil.parser import parse import pytz -# import the logging library -import logging +import shortuuid import pandas as pd from django.db import connections @@ -3209,21 +3209,15 @@ def create(self, request): if error: return Response(error['message'], status=error['status']) - # The root (in the Squonk project) where files will be written. - # This is "understood" by the celery task (which uses this constant). - # e.g. 'fragalysis-files' - transfer_root = settings.SQUONK2_MEDIA_DIRECTORY - logger.info('+ transfer_root=%s', transfer_root) - # Create new file transfer job logger.info('+ Calling ensure_project() to get the Squonk2 Project...') # This requires a Squonk2 Project (created by the Squonk2Agent). # It may be an existing project, or it might be a new project. common_params = CommonParams(user_id=user.id, - access_id=access_id, - target_id=target_id, - session_id=session_project_id) + access_id=access_id, + target_id=target_id, + session_id=session_project_id) sq2_rv = _SQ2A.ensure_project(common_params) if not sq2_rv.success: msg = f'Failed to get/create a Squonk2 Project' \ @@ -3267,6 +3261,10 @@ def create(self, request): job_transfer.transfer_progress = None job_transfer.save() + # The root (in the Squonk project) where files will be written for this Job. + transfer_root = os.path.join(settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path) + logger.info('+ transfer_root=%s', transfer_root) + # Celery/Redis must be running. # This call checks and trys to start them if they're not. assert check_services() From 97c414022ef0ccf1787a9969d0915849dbe6d178 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Fri, 21 Apr 2023 16:09:14 +0200 Subject: [PATCH 110/112] Adds migration --- .../0030_jobfiletransfer_sub_path.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 viewer/migrations/0030_jobfiletransfer_sub_path.py diff --git a/viewer/migrations/0030_jobfiletransfer_sub_path.py b/viewer/migrations/0030_jobfiletransfer_sub_path.py new file mode 100644 index 00000000..58f37c75 --- /dev/null +++ b/viewer/migrations/0030_jobfiletransfer_sub_path.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.14 on 2023-04-21 14:06 + +from django.db import migrations +import shortuuid.django_fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0029_add_job_request_project'), + ] + + operations = [ + migrations.AddField( + model_name='jobfiletransfer', + name='sub_path', + field=shortuuid.django_fields.ShortUUIDField(alphabet='abcdefghijklmnopqrstuvwxyz', length=4, max_length=4, prefix=''), + ), + ] From 67fe46004e82393570527c2071680ddae6c639ce Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 24 Apr 2023 11:16:09 +0100 Subject: [PATCH 111/112] Potential fix for JobFileTransfer sub-path --- README.md | 8 ++++++-- .../0031_fix_JobFileTransfer_sub_path.py | 19 +++++++++++++++++++ viewer/models.py | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 viewer/migrations/0031_fix_JobFileTransfer_sub_path.py diff --git a/README.md b/README.md index e71aa738..9de6e718 100644 --- a/README.md +++ b/README.md @@ -94,14 +94,18 @@ The best approach is to spin-up the development stack (locally) using to make new migrations called "add_job_request_start_and_finish_times" for the viewer's models run the following: - +> Before starting postgres, if you need to, remove any pre-existing Db (if one exists) + with `rm -rf data` + docker-compose up -d docker-compose exec stack bash -Then from within the stack... +Then from within the stack make the migrations. Here we're migrating the `viewer` +application... python manage.py makemigrations viewer --name "add_job_request_start_and_finish_times" -Exit the container and tear-down the deployemnt: - +Exit the container and tear-down the deployment: - docker-compose down diff --git a/viewer/migrations/0031_fix_JobFileTransfer_sub_path.py b/viewer/migrations/0031_fix_JobFileTransfer_sub_path.py new file mode 100644 index 00000000..1d7a35cb --- /dev/null +++ b/viewer/migrations/0031_fix_JobFileTransfer_sub_path.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.14 on 2023-04-24 10:15 + +from django.db import migrations +import shortuuid.django_fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0030_jobfiletransfer_sub_path'), + ] + + operations = [ + migrations.AlterField( + model_name='jobfiletransfer', + name='sub_path', + field=shortuuid.django_fields.ShortUUIDField(alphabet='abcdefghijklmnopqrstuvwxyz', length=4, max_length=4, null=True, prefix=''), + ), + ] diff --git a/viewer/models.py b/viewer/models.py index 4c41a867..40330f04 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -1093,7 +1093,7 @@ class JobFileTransfer(models.Model): snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE) target = models.ForeignKey(Target, null=True, on_delete=models.CASCADE, db_index=True) squonk_project = models.CharField(max_length=200, null=True) - sub_path = ShortUUIDField(length=4, alphabet="abcdefghijklmnopqrstuvwxyz") + sub_path = ShortUUIDField(length=4, alphabet="abcdefghijklmnopqrstuvwxyz", null=True) proteins = models.JSONField(encoder=DjangoJSONEncoder, null=True) # Not used in phase 1 compounds = models.JSONField(encoder=DjangoJSONEncoder, null=True) From 5655579d1abf80f2d274ef9b1c82f0cde1b95693 Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 24 Apr 2023 11:26:06 +0100 Subject: [PATCH 112/112] Better handling of sub_path --- viewer/squonk_job_file_transfer.py | 8 ++++++-- viewer/views.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/viewer/squonk_job_file_transfer.py b/viewer/squonk_job_file_transfer.py index 4625ccd1..26a85301 100644 --- a/viewer/squonk_job_file_transfer.py +++ b/viewer/squonk_job_file_transfer.py @@ -248,9 +248,13 @@ def process_file_transfer(auth_token, trans_dir = create_media_sub_directory(trans_sub_dir) # location in squonk project where files will reside - # e.g. "/fragalysis-files/Mpro" + # e.g. "/fragalysis-files/hjyx/Mpro" for new transfers, + # "/fragalysis-files/Mpro" for existing transfers target = job_transfer.target - squonk_directory = os.path.join('/', settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path, target.title) + if job_transfer.sub_path: + squonk_directory = os.path.join('/', settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path, target.title) + else: + squonk_directory = os.path.join('/', settings.SQUONK2_MEDIA_DIRECTORY, target.title) logger.info('+ Destination squonk_directory=%s', squonk_directory) # This to pick up NULL values from the changeover to using compounds. diff --git a/viewer/views.py b/viewer/views.py index 8592b842..db66df15 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3262,7 +3262,12 @@ def create(self, request): job_transfer.save() # The root (in the Squonk project) where files will be written for this Job. - transfer_root = os.path.join(settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path) + # Something like "fragalysis-files/hjyx" for new transfers, + # "fragalysis-files" for existing transfers + if job_transfer.sub_path: + transfer_root = os.path.join(settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path) + else: + transfer_root = settings.SQUONK2_MEDIA_DIRECTORY logger.info('+ transfer_root=%s', transfer_root) # Celery/Redis must be running.