From d185b291873dd11f065b8b86db6640b9e40448d5 Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 5 Aug 2024 13:23:34 +1000 Subject: [PATCH 01/46] Progress commit for #994 --- .../au/org/ala/ecodata/AssociatedOrg.groovy | 20 ++++++ .../au/org/ala/ecodata/Organisation.groovy | 19 +++++ .../au/org/ala/ecodata/ProjectService.groovy | 4 +- .../5.0/setupAssociatedOrgsForProjects.js | 70 +++++++++++++++++++ scripts/utils/audit.js | 15 ++++ 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 scripts/releases/5.0/setupAssociatedOrgsForProjects.js create mode 100644 scripts/utils/audit.js diff --git a/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy b/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy index eebb4855f..8ff5fe629 100644 --- a/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy +++ b/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy @@ -10,11 +10,29 @@ import groovy.transform.ToString @JsonIgnoreProperties(['metaClass', 'errors', 'expandoMetaClass']) class AssociatedOrg { + /** Reference to the Organisation entity if ecodata has a record of the Organisation */ String organisationId + + /** + * The name of the organisation in the context of the relationship. e.g. it could be a name used + * in a contract with a project that is different from the current business name of the organisation + */ String name String logo String url + /** + * The date the association started. A null date indicates the relationship started at the same + * time as the related entity. e.g. the start of a Project + */ + Date fromDate + + /** + * The date the association e ended. A null date indicates the relationship ended at the same + * time as the related entity. e.g. the end of a Project + */ + Date toDate + /** A description of the association - e.g. Service Provider, Grantee, Sponsor */ String description @@ -27,6 +45,8 @@ class AssociatedOrg { logo nullable: true url nullable: true description nullable: true + fromDate nullable: true + toDate nullable: true } } diff --git a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy index 266d37ac1..fbf534a57 100644 --- a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy @@ -22,6 +22,16 @@ class Organisation { String description String announcements String abn + String abnStatus // N/A, Active, Cancelled + String entityName + String entityType // From ABN register + String businessName + String tradingName + String state + Integer postcode + List externalIds // For financial system vendor codes/reference + List indigenousOrganisationRegistration + List associatedOrgs // e.g. parent organisation such as for NSW LLS group String status = 'active' @@ -42,6 +52,15 @@ class Organisation { announcements nullable: true description nullable: true collectoryInstitutionId nullable: true + abnStatus nullable: true + entityName nullable: true + entityType nullable: true + businessName nullable: true + tradingName nullable: true + state nullable: true + postcode nullable: true + indigenousOrganisationRegistration nullable: true + associatedOrgs nullable: true abn nullable: true hubId nullable: true, validator: { String hubId, Organisation organisation, Errors errors -> GormMongoUtil.validateWriteOnceProperty(organisation, 'organisationId', 'hubId', errors) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 32f627c6d..94f6667c3 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -285,7 +285,9 @@ class ProjectService { if (it.organisationId) { Organisation org = Organisation.findByOrganisationId(it.organisationId) if (org) { - it.name = org.name + if (!it.name) { // Is this going to cause BioCollect an issue? + it.name = org.name + } it.url = org.url it.logo = Document.findByOrganisationIdAndRoleAndStatus(it.organisationId, "logo", ACTIVE)?.thumbnailUrl } diff --git a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js new file mode 100644 index 000000000..bb3cd283b --- /dev/null +++ b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js @@ -0,0 +1,70 @@ +load('../../utils/audit.js'); +let adminUserId = 'system' +let meritHubId = db.hub.findOne({urlPath: "merit"}).hubId; +let projects = db.project.find({hubId:meritHubId, status:{$ne:'deleted'}}); +while (projects.hasNext()) { + let project = projects.next(); + + project.associatedOrgs = []; + + // generally orgs are called service providers in the context of procurements and grantees in the + // context of grants + let description = null; + + let program = db.program.findOne({programId:project.programId}); + if (!program) { + print("No program found for project "+project.projectId+" name:"+project.name); + } + else { + if (program.fundingType) { + switch (program.fundingType) { + case 'SPP': + description = 'Recipient'; + break; + case 'Grant': + description = 'Grantee'; + case 'Procurement': + description = 'Service provider'; + + } + } else if (program.config && program.config.organisationRelationship) { + description = program.config.organisationRelationship; + } + } + + if (!description) { + if (project.plannedStartDate.getTime() > ISODate('2018-07-01T00:00:00+10:00').getDate()) { + description = 'Service provider' + } + else { + description = 'Recipient' + } + } + let associatedOrg = {name:project.organisationName, organisationId:project.organisationId, description:description}; + + if (!associatedOrg.name) { + print("No organisation for project "+project.projectId+" name:"+project.name+" organisationId: "+project.organisationId); + + } + else { + project.associatedOrgs.push(associatedOrg); + + // For now leave these fields as is to not cause issues when switching branches + // and to allow this script to be run repeatedly + //project.organisationId = null; + //project.organisationName = null; + } + + if (project.orgIdSvcProvider) { + let associatedOrg = {name:project.serviceProviderName, organisationId:project.orgIdSvcProvider, description:'Service provider'}; + project.associatedOrgs.push(associatedOrg); + // For now leave these fields as is to not cause issues when switching branches + // and to allow this script to be run repeatedly + //project.orgIdSvcProvider = null; + //project.serviceProviderName = null; + } + + db.project.replaceOne({projectId:project.projectId}, project); + audit(project, project.projectId, 'org.ala.ecodata.Project', adminUserId); + +} \ No newline at end of file diff --git a/scripts/utils/audit.js b/scripts/utils/audit.js new file mode 100644 index 000000000..0f77d026f --- /dev/null +++ b/scripts/utils/audit.js @@ -0,0 +1,15 @@ +/** Inserts a document into the auditMessage collection */ +function audit(entity, entityId, type, userId, projectId, eventType) { + var auditMessage = { + date: ISODate(), + entity: entity, + eventType: eventType || 'Update', + entityType: type, + entityId: entityId, + userId: userId + }; + if (entity.projectId || projectId) { + auditMessage.projectId = (entity.projectId || projectId); + } + db.auditMessage.insertOne(auditMessage); +} \ No newline at end of file From df5ab746e0a927716bbf18b0770f4c1264a5bd06 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 21 Aug 2024 13:52:19 +1000 Subject: [PATCH 02/46] Search associated orgs for project organisationId search #994 --- grails-app/conf/data/mapping.json | 15 +++++----- .../au/org/ala/ecodata/AssociatedOrg.groovy | 4 +++ .../ala/ecodata/OrganisationService.groovy | 1 - .../au/org/ala/ecodata/ProjectService.groovy | 29 ++++++++++++++++--- .../5.0/setupAssociatedOrgsForProjects.js | 21 ++++++++++++++ 5 files changed, 57 insertions(+), 13 deletions(-) diff --git a/grails-app/conf/data/mapping.json b/grails-app/conf/data/mapping.json index 39e8c094a..26fcf577a 100644 --- a/grails-app/conf/data/mapping.json +++ b/grails-app/conf/data/mapping.json @@ -70,10 +70,6 @@ "organisationId": { "type" : "keyword" }, - "orgIdSvcProvider": { - "type" : "keyword", - "copy_to": ["organisationId"] - }, "organisationName": { "type" : "text", "copy_to": ["organisationFacet", "organisationSort"] @@ -81,6 +77,9 @@ "organisationFacet": { "type": "keyword" }, + "linkedOrganisationFacet": { + "type": "keyword" + }, "organisationSort": { "type": "keyword", "normalizer" : "case_insensitive_sort" @@ -92,10 +91,6 @@ "dateCreatedSort" : { "type" : "keyword", "normalizer" : "case_insensitive_sort" }, - "serviceProviderName": { - "type" : "text", - "copy_to": ["organisationName","organisationFacet", "organisationSort"] - }, "associatedOrgs": { "properties" : { "name" : { @@ -109,6 +104,10 @@ "organisationId": { "type": "keyword", "copy_to": ["organisationId"] + }, + "organisationName": { + "type": "keyword", + "copy_to": ["organisationName"] } } }, diff --git a/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy b/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy index 8ff5fe629..901d0017d 100644 --- a/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy +++ b/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy @@ -13,6 +13,8 @@ class AssociatedOrg { /** Reference to the Organisation entity if ecodata has a record of the Organisation */ String organisationId + /** The name of the organisation as referenced via the organisationId */ + String organisationName /** * The name of the organisation in the context of the relationship. e.g. it could be a name used * in a contract with a project that is different from the current business name of the organisation @@ -43,10 +45,12 @@ class AssociatedOrg { organisationId nullable: true name nullable: true logo nullable: true + url nullable: true description nullable: true fromDate nullable: true toDate nullable: true + organisationName nullable: true } } diff --git a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy index f80825d9a..503c5b37e 100644 --- a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy +++ b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy @@ -159,7 +159,6 @@ class OrganisationService { if ('projects' in levelOfDetail) { mapOfProperties.projects = [] mapOfProperties.projects += projectService.search([organisationId: org.organisationId], ['flat']) - mapOfProperties.projects += projectService.search([orgIdSvcProvider: org.organisationId], ['flat']) } if ('documents' in levelOfDetail) { mapOfProperties.documents = documentService.findAllByOwner('organisationId', org.organisationId) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 94f6667c3..56573518c 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -690,15 +690,36 @@ class ProjectService { List search(Map searchCriteria, levelOfDetail = []) { def criteria = Project.createCriteria() + def projects = criteria.list { ne("status", DELETED) searchCriteria.each { prop, value -> + // Special case for organisationId - also included embedded associatedOrg relationships. + if (prop == 'organisationId') { + or { + if (value instanceof List) { + inList(prop, value) + } else { + eq(prop, value) + } - if (value instanceof List) { - inList(prop, value) - } else { - eq(prop, value) + associatedOrgs { + if (value instanceof List) { + inList(prop, value) + } else { + eq(prop, value) + } + } + } + } + else { + if (value instanceof List) { + inList(prop, value) + } else { + eq(prop, value) + } } + } } diff --git a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js index bb3cd283b..85cc58ac1 100644 --- a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js +++ b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js @@ -42,6 +42,17 @@ while (projects.hasNext()) { } let associatedOrg = {name:project.organisationName, organisationId:project.organisationId, description:description}; + if (project.organisationId) { + let organisation = db.organisation.findOne({organisationId:project.organisationId}); + if (!organisation) { + print("OrganisationId "+project.organisationId+" not found for project "+project.projectId+" name:"+project.name); + } + else { + associatedOrg.organisationName = organisation.name; + } + } + + if (!associatedOrg.name) { print("No organisation for project "+project.projectId+" name:"+project.name+" organisationId: "+project.organisationId); @@ -57,6 +68,16 @@ while (projects.hasNext()) { if (project.orgIdSvcProvider) { let associatedOrg = {name:project.serviceProviderName, organisationId:project.orgIdSvcProvider, description:'Service provider'}; + + let organisation = db.organisation.findOne({organisationId:project.orgIdSvcProvider}); + if (!organisation) { + print("OrganisationId "+project.orgIdSvcProvider+" not found for project "+project.projectId+" name:"+project.name); + } + else { + associatedOrg.organisationName = organisation.name; + } + + project.associatedOrgs.push(associatedOrg); // For now leave these fields as is to not cause issues when switching branches // and to allow this script to be run repeatedly From f51472f4892998216401b349c74fa508b46da50a Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 21 Aug 2024 14:44:42 +1000 Subject: [PATCH 03/46] Updated tests #994 --- .../au/org/ala/ecodata/OrganisationControllerSpec.groovy | 4 ++-- .../org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy index d6ce0b57e..36d7b3064 100644 --- a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy +++ b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy @@ -119,14 +119,14 @@ class OrganisationControllerSpec extends IntegrationTestHelper { } - void "projects can be associated with an organisation by the serviceProviderOrganisationId property"() { + void "projects can be associated with an organisation by the associatedOrgs property"() { setup: // Create some data for the database. def organisation = TestDataHelper.buildOrganisation([name: 'org 1']) def projects = [] (1..2).each { - projects << TestDataHelper.buildProject([orgIdSvcProvider: organisation.organisationId, name:'svc project '+it]) + projects << TestDataHelper.buildProject([associatedOrgs: [[organisationId:organisation.organisationId, name:'org project '+it]]]) } projects << TestDataHelper.buildProject([organisationId: organisation.organisationId, name:'org project']) (1..3).each { diff --git a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy index 4e7688fcd..c087e7577 100644 --- a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy +++ b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy @@ -112,7 +112,7 @@ class OrganisationServiceIntegrationSpec extends IntegrationTestHelper { } - void "projects can be associated with an organisation by the serviceProviderOrganisationId property"() { + void "projects can be associated with an organisation by the associatedOrgs property"() { setup: def organisation @@ -122,7 +122,7 @@ class OrganisationServiceIntegrationSpec extends IntegrationTestHelper { organisation = TestDataHelper.buildOrganisation([name: 'Test Organisation2']) def projects = [] (1..2).each { - projects << TestDataHelper.buildProject([orgIdSvcProvider: organisation.organisationId]) + projects << TestDataHelper.buildProject([associatedOrgs: [[organisationId:organisation.organisationId, name:'org project '+it]]]) } projects << TestDataHelper.buildProject([organisationId: organisation.organisationId]) (1..3).each { From 7852d3370db59b41c386dacc3b0be8dca160607b Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 26 Aug 2024 14:53:57 +1000 Subject: [PATCH 04/46] Added business/contract names #994 --- grails-app/domain/au/org/ala/ecodata/Organisation.groovy | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy index fbf534a57..d077cea0a 100644 --- a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy @@ -25,14 +25,13 @@ class Organisation { String abnStatus // N/A, Active, Cancelled String entityName String entityType // From ABN register - String businessName - String tradingName + List businessNames String state Integer postcode List externalIds // For financial system vendor codes/reference List indigenousOrganisationRegistration List associatedOrgs // e.g. parent organisation such as for NSW LLS group - + List contractNames // When contracts are written for projects with this organisation with a name that doesn't match the organisation name String status = 'active' String collectoryInstitutionId // Reference to the Collectory @@ -55,8 +54,8 @@ class Organisation { abnStatus nullable: true entityName nullable: true entityType nullable: true - businessName nullable: true - tradingName nullable: true + businessNames nullable: true + contractNames nullable: true state nullable: true postcode nullable: true indigenousOrganisationRegistration nullable: true From 71707f696a811c23693467eb15d3b15a48a01d0c Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 28 Aug 2024 08:37:53 +1000 Subject: [PATCH 05/46] Changed OrganisationService to use data binding #994 --- .../au/org/ala/ecodata/AssociatedOrg.groovy | 3 -- .../au/org/ala/ecodata/Organisation.groovy | 16 ++++++++-- .../ala/ecodata/OrganisationService.groovy | 31 ++++++++++++------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy b/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy index 901d0017d..f052ad682 100644 --- a/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy +++ b/grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy @@ -13,8 +13,6 @@ class AssociatedOrg { /** Reference to the Organisation entity if ecodata has a record of the Organisation */ String organisationId - /** The name of the organisation as referenced via the organisationId */ - String organisationName /** * The name of the organisation in the context of the relationship. e.g. it could be a name used * in a contract with a project that is different from the current business name of the organisation @@ -50,7 +48,6 @@ class AssociatedOrg { description nullable: true fromDate nullable: true toDate nullable: true - organisationName nullable: true } } diff --git a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy index d077cea0a..635282c83 100644 --- a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy @@ -22,9 +22,12 @@ class Organisation { String description String announcements String abn + String url String abnStatus // N/A, Active, Cancelled String entityName - String entityType // From ABN register + String sourceSystem // MERIT or Collectory + String entityType // Type code from the ABN register + String orgType // Type name as selected in BioCollect/ Name from the ABN register List businessNames String state Integer postcode @@ -32,16 +35,21 @@ class Organisation { List indigenousOrganisationRegistration List associatedOrgs // e.g. parent organisation such as for NSW LLS group List contractNames // When contracts are written for projects with this organisation with a name that doesn't match the organisation name - String status = 'active' + String status = Status.ACTIVE + + /** Stores configuration information for how reports should be generated for this organisation (if applicable) */ + Map config String collectoryInstitutionId // Reference to the Collectory Date dateCreated Date lastUpdated + static embedded = ['externalIds', 'associatedOrgs'] static mapping = { organisationId index: true + name index:true version false } @@ -54,6 +62,7 @@ class Organisation { abnStatus nullable: true entityName nullable: true entityType nullable: true + orgType nullable: true businessNames nullable: true contractNames nullable: true state nullable: true @@ -61,6 +70,9 @@ class Organisation { indigenousOrganisationRegistration nullable: true associatedOrgs nullable: true abn nullable: true + url nullable: true + config nullable: true + sourceSystem nullable: true hubId nullable: true, validator: { String hubId, Organisation organisation, Errors errors -> GormMongoUtil.validateWriteOnceProperty(organisation, 'organisationId', 'hubId', errors) } diff --git a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy index a8283b687..c4d1a3479 100644 --- a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy +++ b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy @@ -3,6 +3,7 @@ package au.org.ala.ecodata import com.mongodb.client.MongoCollection import com.mongodb.client.model.Filters import grails.validation.ValidationException +import grails.web.databinding.DataBinder import org.bson.conversions.Bson import static au.org.ala.ecodata.Status.DELETED @@ -10,11 +11,13 @@ import static au.org.ala.ecodata.Status.DELETED /** * Works with Organisations, mostly CRUD operations at this point. */ -class OrganisationService { +class OrganisationService implements DataBinder { /** Use to include related projects in the toMap method */ public static final String PROJECTS = 'projects' + private static final List EXCLUDE_FROM_BINDING = ['organisationId', 'collectoryInstitutionId', 'status', 'id'] + static transactional = 'mongo' static final FLAT = 'flat' @@ -40,7 +43,7 @@ class OrganisationService { } def list(levelOfDetail = []) { - return Organisation.findAllByStatusNotEqual('deleted').collect{toMap(it, levelOfDetail)} + return Organisation.findAllByStatusNotEqual(DELETED).collect{toMap(it, levelOfDetail)} } def create(Map props, boolean createInCollectory = true) { @@ -51,12 +54,8 @@ class OrganisationService { organisation.collectoryInstitutionId = createCollectoryInstitution(props) } try { - // name is a mandatory property and hence needs to be set before dynamic properties are used (as they trigger validations) + bindData(organisation, props, [exclude:EXCLUDE_FROM_BINDING]) organisation.save(failOnError: true, flush:true) - props.remove('id') - props.remove('organisationId') - props.remove('collectoryInstitutionId') - commonService.updateProperties(organisation, props) // Assign the creating user as an admin. permissionService.addUserAsRoleToOrganisation(userService.getCurrentUserDetails()?.userId, AccessLevel.admin, organisation.organisationId) @@ -97,17 +96,25 @@ class OrganisationService { if (organisation) { try { - String oldName = organisation.name - commonService.updateProperties(organisation, props) // if no collectory institution exists for this organisation, create one + // We shouldn't be doing this unless the org is attached to a project that exports data + // to the ALA. if (!organisation.collectoryInstitutionId || organisation.collectoryInstitutionId == 'null' || organisation.collectoryInstitutionId == '') { - props.collectoryInstitutionId = createCollectoryInstitution(props) + organisation.collectoryInstitutionId = createCollectoryInstitution(props) } +œ + String oldName = organisation.name + bindData(organisation, props, [exclude:EXCLUDE_FROM_BINDING]) - getCommonService().updateProperties(organisation, props) if (props.name && (oldName != props.name)) { projectService.updateOrganisationName(organisation.organisationId, props.name) } + props.contractNames?.each { + if (!it in organisation.contractNames) { + + } + } + organisation.save(failOnError:true) return [status:'ok'] } catch (Exception e) { Organisation.withSession { session -> session.clear() } @@ -136,7 +143,7 @@ class OrganisationService { if (destroy) { organisation.delete() } else { - organisation.status = 'deleted' + organisation.status = DELETED organisation.save(flush: true, failOnError: true) } return [status: 'ok'] From d2e7dbd3778b2e53e661ba88b8a2a304f9018a2e Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 28 Aug 2024 08:57:28 +1000 Subject: [PATCH 06/46] Remove dynamic property check from test #994 --- .../ecodata/OrganisationServiceIntegrationSpec.groovy | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy index c087e7577..56fe95365 100644 --- a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy +++ b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy @@ -55,18 +55,17 @@ class OrganisationServiceIntegrationSpec extends IntegrationTestHelper { Organisation.withTransaction { savedOrganisation = organisationService.get(organisationId) } - //organisationController.response.reset() - // organisationController.get(organisationId) - - // def savedOrganisation = extractJson(organisationController.response.text) then: "ensure the properties are the same as the original" savedOrganisation.organisationId == organisationId savedOrganisation.name == org.name savedOrganisation.description == org.description - savedOrganisation.dynamicProperty == org.dynamicProperty savedOrganisation.collectoryInstitutionId == institutionId + and: "The OrganisationService no longer supports dynamic properties" + savedOrganisation.dynamicProperty == null + + and: "the user who created the organisation is an admin of the new organisation" def orgPermissions = UserPermission.findAllByEntityIdAndEntityType(savedOrganisation.organisationId, Organisation.class.name) orgPermissions.size() == 1 From 8e638cab037a78ea08e2c71cd86a21328f401f13 Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 9 Sep 2024 15:12:04 +1000 Subject: [PATCH 07/46] Update org names when contract names changed #994 --- .../ala/ecodata/OrganisationService.groovy | 9 +++-- .../au/org/ala/ecodata/ProjectService.groovy | 26 +++++++++++--- .../5.0/setupAssociatedOrgsForProjects.js | 3 +- .../org/ala/ecodata/ProjectServiceSpec.groovy | 35 +++++++++++++++++++ 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy index c4d1a3479..053a03c14 100644 --- a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy +++ b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy @@ -104,15 +104,14 @@ class OrganisationService implements DataBinder { } œ String oldName = organisation.name + List contractNameChanges = props.remove('contractNameChanges') bindData(organisation, props, [exclude:EXCLUDE_FROM_BINDING]) if (props.name && (oldName != props.name)) { - projectService.updateOrganisationName(organisation.organisationId, props.name) + projectService.updateOrganisationName(organisation.organisationId, oldName, props.name) } - props.contractNames?.each { - if (!it in organisation.contractNames) { - - } + contractNameChanges?.each { Map change -> + projectService.updateOrganisationName(organisation.organisationId, change.oldName, change.newName) } organisation.save(failOnError:true) return [status:'ok'] diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 433c94bb0..92483046e 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -743,10 +743,28 @@ class ProjectService { * @param orgId identifies the organsation that has changed name * @param orgName the new organisation name */ - void updateOrganisationName(orgId, orgName) { - Project.findAllByOrganisationIdAndStatusNotEqual(orgId, DELETED).each { project -> - project.organisationName = orgName - project.save() + void updateOrganisationName(String orgId, String oldName, String newName) { + Project.findAllByOrganisationIdAndOrganisationNameAndStatusNotEqual(orgId, oldName, DELETED).each { project -> + project.organisationName = newName + project.save(flush:true) + } + + List projects = Project.where { + status != DELETED + associatedOrgs { + organisationId == orgId + name == oldName + } + }.list() + + + projects?.each { Project project -> + project.associatedOrgs.each { org -> + if (org.organisationId == orgId && org.name == oldName) { + org.name = newName + } + } + project.save(flush:true) } } diff --git a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js index 85cc58ac1..e0268d539 100644 --- a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js +++ b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js @@ -22,7 +22,8 @@ while (projects.hasNext()) { description = 'Recipient'; break; case 'Grant': - description = 'Grantee'; + description = 'Recipient'; + break; case 'Procurement': description = 'Service provider'; diff --git a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy index 7924276c7..5f134a0a6 100644 --- a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy @@ -1064,4 +1064,39 @@ class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest Date: Mon, 16 Sep 2024 15:49:58 +1000 Subject: [PATCH 08/46] Disable org collectory sync by default #2382 --- .../services/au/org/ala/ecodata/OrganisationService.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy index 053a03c14..a6031f1b1 100644 --- a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy +++ b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy @@ -46,7 +46,7 @@ class OrganisationService implements DataBinder { return Organisation.findAllByStatusNotEqual(DELETED).collect{toMap(it, levelOfDetail)} } - def create(Map props, boolean createInCollectory = true) { + def create(Map props, boolean createInCollectory = false) { def organisation = new Organisation(organisationId: Identifiers.getNew(true, ''), name:props.name) @@ -90,7 +90,7 @@ class OrganisationService implements DataBinder { return institutionId } - def update(String id, props) { + def update(String id, props, boolean createInCollectory = false) { def organisation = Organisation.findByOrganisationId(id) if (organisation) { @@ -99,7 +99,7 @@ class OrganisationService implements DataBinder { // if no collectory institution exists for this organisation, create one // We shouldn't be doing this unless the org is attached to a project that exports data // to the ALA. - if (!organisation.collectoryInstitutionId || organisation.collectoryInstitutionId == 'null' || organisation.collectoryInstitutionId == '') { + if (createInCollectory && (!organisation.collectoryInstitutionId || organisation.collectoryInstitutionId == 'null' || organisation.collectoryInstitutionId == '')) { organisation.collectoryInstitutionId = createCollectoryInstitution(props) } œ From a83a3f76b4d17a6af873113da1b5fd7fb41fffe5 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 17 Sep 2024 11:43:34 +1000 Subject: [PATCH 09/46] Ensure BioCollect associated orgs use org name to preserve behaviour #994 --- .../releases/5.0/updateAssociatedOrgNames.js | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 scripts/releases/5.0/updateAssociatedOrgNames.js diff --git a/scripts/releases/5.0/updateAssociatedOrgNames.js b/scripts/releases/5.0/updateAssociatedOrgNames.js new file mode 100644 index 000000000..43ed54988 --- /dev/null +++ b/scripts/releases/5.0/updateAssociatedOrgNames.js @@ -0,0 +1,30 @@ +load('../../utils/audit.js'); +let projects = db.project.find({status:{$ne:'deleted'}, associatedOrgs:{$exists:true}, isMERIT:false}); +while (projects.hasNext()) { + let changed = false; + + let project = projects.next(); + let associatedOrgs = project.associatedOrgs; + if (associatedOrgs) { + for (let i = 0; i < associatedOrgs.length; i++) { + if (associatedOrgs[i].organisationId) { + let org = db.organisation.findOne({organisationId: associatedOrgs[i].organisationId}); + if (org) { + if (org.name != associatedOrgs[i].name) { + print("Updating associated org for project " + project.projectId + " from " + associatedOrgs[i].name + " to " + org.name); + associatedOrgs[i].name = org.name; + changed = true; + } + } else { + print("No organisation found for associated org " + associatedOrgs[i].organisationId + " in project " + project.projectId); + } + + } + } + if (changed) { + db.project.replaceOne({projectId: project.projectId}, project); + audit(project, project.projectId, 'au.org.ala.ecodata.Project', 'system'); + } + + } +} \ No newline at end of file From e62a3565806d844e06366a5fe9ab5e9e973dd9a6 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 17 Sep 2024 13:59:15 +1000 Subject: [PATCH 10/46] Updated tests #994 --- .../ecodata/OrganisationServiceSpec.groovy | 60 +------------------ 1 file changed, 2 insertions(+), 58 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/OrganisationServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/OrganisationServiceSpec.groovy index 95275f532..aeaabf062 100644 --- a/src/test/groovy/au/org/ala/ecodata/OrganisationServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/OrganisationServiceSpec.groovy @@ -42,7 +42,7 @@ class OrganisationServiceSpec extends Specification implements ServiceUnitTest> projects - projectService.search([orgIdSvcProvider: orgId]) >> [] - - - when: - def result - // print (orgId) - // Organisation.withNewTransaction { - result = service.get(orgId) - // print result - // } - // def result = service.toMap(org) - - then: - result.organisationId == orgId - result.name == org.name - result.description == org.description - result.projects == null - - when: - // Organisation.withNewTransaction { - // result = service.get(orgId, [OrganisationService.PROJECTS]) - // print result - // } - def result1 = service.toMap(org, [OrganisationService.PROJECTS]) - - then: - result1.organisationId == orgId - result1.name == org.name - result1.description == org.description - result1.dynamicProperty == org['dynamicProperty'] - result1.projects == projects - - } -*/ - - - - } From 3fc5fa9600c832ce405050108b76b30571280319 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 17 Sep 2024 14:22:52 +1000 Subject: [PATCH 11/46] Updated tests #994 --- .../groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy index 36d7b3064..c989089f1 100644 --- a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy +++ b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationControllerSpec.groovy @@ -75,8 +75,6 @@ class OrganisationControllerSpec extends IntegrationTestHelper { savedOrganisation.organisationId == organisationId savedOrganisation.name == org.name savedOrganisation.description == org.description - // savedOrganisation.dynamicProperty == org.dynamicProperty (dynamic properties not working in tests) - savedOrganisation.collectoryInstitutionId == institutionId and: "the user who created the organisation is an admin of the new organisation" def orgPermissions = UserPermission.findAllByEntityIdAndEntityType(savedOrganisation.organisationId, Organisation.class.name) From 9f470e4ecae49045fd0d676ccc5c8836dad68d18 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 17 Sep 2024 14:54:30 +1000 Subject: [PATCH 12/46] Updated tests #994 --- .../org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy index 56fe95365..14d565659 100644 --- a/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy +++ b/src/integration-test/groovy/au/org/ala/ecodata/OrganisationServiceIntegrationSpec.groovy @@ -41,7 +41,7 @@ class OrganisationServiceIntegrationSpec extends IntegrationTestHelper { // setupPost(organisationController.request, org) when: "creating an organisation" - def result = organisationService.create(org) + def result = organisationService.create(org, true) then: "ensure we get a response including an organisationId" // def resp = extractJson(organisationController.response.text) From 6d9552adcd4307409f35a6c4fc9bf0ed3513453e Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 3 Oct 2024 17:10:42 +1000 Subject: [PATCH 13/46] 5.1-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 51515114d..2c18d5340 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.0.1" +version "5.1-SNAPSHOT" group "au.org.ala" description "Ecodata" From 10310f9a05e6c08869ff68a752c16323b35b0ef9 Mon Sep 17 00:00:00 2001 From: chrisala Date: Fri, 4 Oct 2024 09:44:25 +1000 Subject: [PATCH 14/46] Use "Recipient" for default org relationship #994 --- .../5.0/setupAssociatedOrgsForProjects.js | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js index e0268d539..6dba6cf90 100644 --- a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js +++ b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js @@ -9,38 +9,7 @@ while (projects.hasNext()) { // generally orgs are called service providers in the context of procurements and grantees in the // context of grants - let description = null; - - let program = db.program.findOne({programId:project.programId}); - if (!program) { - print("No program found for project "+project.projectId+" name:"+project.name); - } - else { - if (program.fundingType) { - switch (program.fundingType) { - case 'SPP': - description = 'Recipient'; - break; - case 'Grant': - description = 'Recipient'; - break; - case 'Procurement': - description = 'Service provider'; - - } - } else if (program.config && program.config.organisationRelationship) { - description = program.config.organisationRelationship; - } - } - - if (!description) { - if (project.plannedStartDate.getTime() > ISODate('2018-07-01T00:00:00+10:00').getDate()) { - description = 'Service provider' - } - else { - description = 'Recipient' - } - } + let description = 'Recipient'; let associatedOrg = {name:project.organisationName, organisationId:project.organisationId, description:description}; if (project.organisationId) { From 5af34df5a35e786e1955f7f15857f695f90a8b01 Mon Sep 17 00:00:00 2001 From: chrisala Date: Fri, 4 Oct 2024 16:06:59 +1000 Subject: [PATCH 15/46] Moved script to merit #994 --- .../5.0/setupAssociatedOrgsForProjects.js | 61 ------------------- 1 file changed, 61 deletions(-) delete mode 100644 scripts/releases/5.0/setupAssociatedOrgsForProjects.js diff --git a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js b/scripts/releases/5.0/setupAssociatedOrgsForProjects.js deleted file mode 100644 index 6dba6cf90..000000000 --- a/scripts/releases/5.0/setupAssociatedOrgsForProjects.js +++ /dev/null @@ -1,61 +0,0 @@ -load('../../utils/audit.js'); -let adminUserId = 'system' -let meritHubId = db.hub.findOne({urlPath: "merit"}).hubId; -let projects = db.project.find({hubId:meritHubId, status:{$ne:'deleted'}}); -while (projects.hasNext()) { - let project = projects.next(); - - project.associatedOrgs = []; - - // generally orgs are called service providers in the context of procurements and grantees in the - // context of grants - let description = 'Recipient'; - let associatedOrg = {name:project.organisationName, organisationId:project.organisationId, description:description}; - - if (project.organisationId) { - let organisation = db.organisation.findOne({organisationId:project.organisationId}); - if (!organisation) { - print("OrganisationId "+project.organisationId+" not found for project "+project.projectId+" name:"+project.name); - } - else { - associatedOrg.organisationName = organisation.name; - } - } - - - if (!associatedOrg.name) { - print("No organisation for project "+project.projectId+" name:"+project.name+" organisationId: "+project.organisationId); - - } - else { - project.associatedOrgs.push(associatedOrg); - - // For now leave these fields as is to not cause issues when switching branches - // and to allow this script to be run repeatedly - //project.organisationId = null; - //project.organisationName = null; - } - - if (project.orgIdSvcProvider) { - let associatedOrg = {name:project.serviceProviderName, organisationId:project.orgIdSvcProvider, description:'Service provider'}; - - let organisation = db.organisation.findOne({organisationId:project.orgIdSvcProvider}); - if (!organisation) { - print("OrganisationId "+project.orgIdSvcProvider+" not found for project "+project.projectId+" name:"+project.name); - } - else { - associatedOrg.organisationName = organisation.name; - } - - - project.associatedOrgs.push(associatedOrg); - // For now leave these fields as is to not cause issues when switching branches - // and to allow this script to be run repeatedly - //project.orgIdSvcProvider = null; - //project.serviceProviderName = null; - } - - db.project.replaceOne({projectId:project.projectId}, project); - audit(project, project.projectId, 'org.ala.ecodata.Project', adminUserId); - -} \ No newline at end of file From ae87e041a2b4604e704bfdf00272c31b73fe647d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 20:01:17 +0000 Subject: [PATCH 16/46] Bump cookie and socket.io Bumps [cookie](https://github.com/jshttp/cookie) and [socket.io](https://github.com/socketio/socket.io). These dependencies needed to be updated together. Updates `cookie` from 0.4.2 to 0.7.2 - [Release notes](https://github.com/jshttp/cookie/releases) - [Commits](https://github.com/jshttp/cookie/compare/v0.4.2...v0.7.2) Updates `socket.io` from 4.7.5 to 4.8.0 - [Release notes](https://github.com/socketio/socket.io/releases) - [Changelog](https://github.com/socketio/socket.io/blob/main/CHANGELOG.md) - [Commits](https://github.com/socketio/socket.io/compare/socket.io@4.7.5...socket.io@4.8.0) --- updated-dependencies: - dependency-name: cookie dependency-type: indirect - dependency-name: socket.io dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 56 +++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7a7183dec..6158a3e9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1317,9 +1317,9 @@ } }, "node_modules/cookie": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", - "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "dev": true, "engines": { "node": ">= 0.6" @@ -1543,9 +1543,9 @@ } }, "node_modules/engine.io": { - "version": "6.5.5", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.5.5.tgz", - "integrity": "sha512-C5Pn8Wk+1vKBoHghJODM63yk8MvrO9EWZUfkAt5HAqIgPE4/8FF0PEGHXtEd40l223+cE5ABWuPzm38PHFXfMA==", + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.2.tgz", + "integrity": "sha512-gmNvsYi9C8iErnZdVcJnvCpSKbWTt1E8+JZo8b+daLninywUWi5NQ5STSHZ9rFjFO7imNcvb8Pc5pe/wMR5xEw==", "dev": true, "dependencies": { "@types/cookie": "^0.4.1", @@ -1553,7 +1553,7 @@ "@types/node": ">=10.0.0", "accepts": "~1.3.4", "base64id": "2.0.0", - "cookie": "~0.4.1", + "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.3.1", "engine.io-parser": "~5.2.1", @@ -1564,9 +1564,9 @@ } }, "node_modules/engine.io-parser": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.2.tgz", - "integrity": "sha512-RcyUFKA93/CXH20l4SoVvzZfrSDMOTUS3bWVpTt2FuFP+XYrL8i8oonHP7WInRyVHXh0n/ORtoeiE1os+8qkSw==", + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz", + "integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==", "dev": true, "engines": { "node": ">=10.0.0" @@ -3673,16 +3673,16 @@ } }, "node_modules/socket.io": { - "version": "4.7.5", - "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.7.5.tgz", - "integrity": "sha512-DmeAkF6cwM9jSfmp6Dr/5/mfMwb5Z5qRrSXLpo3Fq5SqyU8CMF15jIN4ZhfSwu35ksM1qmHZDQ/DK5XTccSTvA==", + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.0.tgz", + "integrity": "sha512-8U6BEgGjQOfGz3HHTYaC/L1GaxDCJ/KM0XTkJly0EhZ5U/du9uNEZy4ZgYzEzIqlx2CMm25CrCqr1ck899eLNA==", "dev": true, "dependencies": { "accepts": "~1.3.4", "base64id": "~2.0.0", "cors": "~2.8.5", "debug": "~4.3.2", - "engine.io": "~6.5.2", + "engine.io": "~6.6.0", "socket.io-adapter": "~2.5.2", "socket.io-parser": "~4.2.4" }, @@ -5404,9 +5404,9 @@ } }, "cookie": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", - "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "dev": true }, "cors": { @@ -5571,9 +5571,9 @@ } }, "engine.io": { - "version": "6.5.5", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.5.5.tgz", - "integrity": "sha512-C5Pn8Wk+1vKBoHghJODM63yk8MvrO9EWZUfkAt5HAqIgPE4/8FF0PEGHXtEd40l223+cE5ABWuPzm38PHFXfMA==", + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.2.tgz", + "integrity": "sha512-gmNvsYi9C8iErnZdVcJnvCpSKbWTt1E8+JZo8b+daLninywUWi5NQ5STSHZ9rFjFO7imNcvb8Pc5pe/wMR5xEw==", "dev": true, "requires": { "@types/cookie": "^0.4.1", @@ -5581,7 +5581,7 @@ "@types/node": ">=10.0.0", "accepts": "~1.3.4", "base64id": "2.0.0", - "cookie": "~0.4.1", + "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.3.1", "engine.io-parser": "~5.2.1", @@ -5589,9 +5589,9 @@ } }, "engine.io-parser": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.2.tgz", - "integrity": "sha512-RcyUFKA93/CXH20l4SoVvzZfrSDMOTUS3bWVpTt2FuFP+XYrL8i8oonHP7WInRyVHXh0n/ORtoeiE1os+8qkSw==", + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz", + "integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==", "dev": true }, "enhanced-resolve": { @@ -7147,16 +7147,16 @@ } }, "socket.io": { - "version": "4.7.5", - "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.7.5.tgz", - "integrity": "sha512-DmeAkF6cwM9jSfmp6Dr/5/mfMwb5Z5qRrSXLpo3Fq5SqyU8CMF15jIN4ZhfSwu35ksM1qmHZDQ/DK5XTccSTvA==", + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.0.tgz", + "integrity": "sha512-8U6BEgGjQOfGz3HHTYaC/L1GaxDCJ/KM0XTkJly0EhZ5U/du9uNEZy4ZgYzEzIqlx2CMm25CrCqr1ck899eLNA==", "dev": true, "requires": { "accepts": "~1.3.4", "base64id": "~2.0.0", "cors": "~2.8.5", "debug": "~4.3.2", - "engine.io": "~6.5.2", + "engine.io": "~6.6.0", "socket.io-adapter": "~2.5.2", "socket.io-parser": "~4.2.4" } From 480a254c3183f8fb85e92c9c58c7c3b112cd1121 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 15 Oct 2024 14:35:18 +1100 Subject: [PATCH 17/46] Use associatedOrgs for project organisation data on export #994 --- .../reporting/ProjectXlsExporter.groovy | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index 0c2b060b4..a702f2d13 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -1,6 +1,7 @@ package au.org.ala.ecodata.reporting import au.org.ala.ecodata.ActivityForm +import au.org.ala.ecodata.AssociatedOrg import au.org.ala.ecodata.DataDescription import au.org.ala.ecodata.ExternalId import au.org.ala.ecodata.ManagementUnit @@ -38,11 +39,11 @@ class ProjectXlsExporter extends ProjectExporter { List configurableIntersectionHeaders = getIntersectionHeaders() List configurableIntersectionProperties = getIntersectionProperties() - List commonProjectHeadersWithoutSites = ['Project ID', 'Grant ID', 'External ID', 'Internal order number', 'Work order id', 'Organisation', 'Service Provider', 'Management Unit', 'Name', 'Description', 'Program', 'Sub-program', 'Start Date', 'End Date', 'Contracted Start Date', 'Contracted End Date', 'Funding', 'Funding Type', 'Status', "Last Modified"] + configurableIntersectionHeaders - List commonProjectPropertiesRaw = ['grantId', 'externalId', 'internalOrderId', 'workOrderId', 'organisationName', 'serviceProviderName', 'managementUnitName', 'name', 'description', 'associatedProgram', 'associatedSubProgram', 'plannedStartDate', 'plannedEndDate', 'contractStartDate', 'contractEndDate', 'funding', 'fundingType', 'status', 'lastUpdated'] + configurableIntersectionProperties + List commonProjectHeadersWithoutSites = ['Project ID', 'Grant ID', 'External ID', 'Internal order number', 'Work order id', 'Recipient (Contract name)', 'Recipient (ID)', 'Management Unit', 'Name', 'Description', 'Program', 'Sub-program', 'Start Date', 'End Date', 'Contracted Start Date', 'Contracted End Date', 'Funding', 'Funding Type', 'Status', "Last Modified"] + configurableIntersectionHeaders + List commonProjectPropertiesRaw = ['grantId', 'externalId', 'internalOrderId', 'workOrderId', 'organisationName', 'organisationId', 'managementUnitName', 'name', 'description', 'associatedProgram', 'associatedSubProgram', 'plannedStartDate', 'plannedEndDate', 'contractStartDate', 'contractEndDate', 'funding', 'fundingType', 'status', 'lastUpdated'] + configurableIntersectionProperties - List projectHeadersWithTerminationReason = ['Project ID', 'Grant ID', 'External ID', 'Internal order number', 'Work order id', 'Organisation', 'Service Provider', 'Management Unit', 'Name', 'Description', 'Program', 'Sub-program', 'Start Date', 'End Date', 'Contracted Start Date', 'Contracted End Date', 'Funding', 'Funding Type', 'Status'] + configurableIntersectionHeaders + ['Termination Reason', 'Last Modified'] - List projectPropertiesTerminationReason = ['grantId', 'externalId', 'internalOrderId', 'workOrderId', 'organisationName', 'serviceProviderName', 'managementUnitName', 'name', 'description', 'associatedProgram', 'associatedSubProgram', 'plannedStartDate', 'plannedEndDate', 'contractStartDate', 'contractEndDate', 'funding', 'fundingType', 'status'] + configurableIntersectionProperties + List projectHeadersWithTerminationReason = ['Project ID', 'Grant ID', 'External ID', 'Internal order number', 'Work order id', 'Recipient (Contract name)', 'Recipient (ID)', 'Management Unit', 'Name', 'Description', 'Program', 'Sub-program', 'Start Date', 'End Date', 'Contracted Start Date', 'Contracted End Date', 'Funding', 'Funding Type', 'Status'] + configurableIntersectionHeaders + ['Termination Reason', 'Last Modified'] + List projectPropertiesTerminationReason = ['grantId', 'externalId', 'internalOrderId', 'workOrderId', 'organisationName', 'organisationId', 'managementUnitName', 'name', 'description', 'associatedProgram', 'associatedSubProgram', 'plannedStartDate', 'plannedEndDate', 'contractStartDate', 'contractEndDate', 'funding', 'fundingType', 'status'] + configurableIntersectionProperties List projectPropertiesWithTerminationReason = ['projectId'] + projectPropertiesTerminationReason.collect{PROJECT_DATA_PREFIX+it} + ["terminationReason", PROJECT_DATA_PREFIX+"lastUpdated"] @@ -55,8 +56,13 @@ class ProjectXlsExporter extends ProjectExporter { List commonProjectHeaders = commonProjectHeadersWithoutSites + stateHeaders + electorateHeaders + projectApprovalHeaders List commonProjectProperties = commonProjectPropertiesWithoutSites + stateProperties + electorateProperties + projectApprovalProperties - List projectHeaders = projectHeadersWithTerminationReason + projectStateHeaders - List projectProperties = projectPropertiesWithTerminationReason + projectStateProperties + List associatedOrgProjectHeaders = (1..3).collect{['Contract name '+it, 'Organisation ID '+it, 'Organisation relationship from date '+it, 'Organisation relationship to date '+it, 'Organisation relationship '+it]}.flatten() + List associatedOrgProperties = ['name', 'organisationId', 'fromDate', 'toDate', 'description'] + + List associatedOrgProjectProperties = (1..3).collect{['associatedOrg_name'+it, 'associatedOrg_organisationId'+it, 'associatedOrg_fromDate'+it, 'associatedOrg_toDate'+it, 'associatedOrg_description'+it]}.flatten() + + List projectHeaders = projectHeadersWithTerminationReason + associatedOrgProjectHeaders + projectStateHeaders + List projectProperties = projectPropertiesWithTerminationReason + associatedOrgProjectProperties + projectStateProperties List siteStateHeaders = (1..5).collect{'State '+it} @@ -164,6 +170,7 @@ class ProjectXlsExporter extends ProjectExporter { List rdpMonitoringIndicatorsHeaders =commonProjectHeaders + ['Code', 'Monitoring methodology', 'Project service / Target measure/s', 'Monitoring method', 'Evidence to be retained'] List rdpMonitoringIndicatorsProperties =commonProjectProperties + ['relatedBaseline', 'data1', 'relatedTargetMeasures','protocols', 'evidence'] + OutputModelProcessor processor = new OutputModelProcessor() ProjectService projectService @@ -320,6 +327,14 @@ class ProjectXlsExporter extends ProjectExporter { if (project.managementUnitId) { project[PROJECT_DATA_PREFIX+'managementUnitName'] = managementUnitNames[project.managementUnitId] } + + Date now = new Date() + List orgs = project.associatedOrgs?.findAll{(!it.fromDate || it.fromDate <= now) && (!it.toDate || it.toDate >= now)} + if (orgs) { + project.organisationName = orgs[0].name + project.organisationId = orgs[0].organisationId + } + filterExternalIds(project, PROJECT_DATA_PREFIX) } @@ -499,6 +514,13 @@ class ProjectXlsExporter extends ProjectExporter { project[electorate] = projectElectorates.contains(electorate) ? 'Y' : 'N' } + project.associatedOrgs?.eachWithIndex { org, i -> + Map orgProps = associatedOrgProperties.collectEntries{ + [('associatedOrg_'+it+(i+1)):org[it]] + } + project.putAll(orgProps) + } + projectSheet.add([project], properties, row + 1) } From a6c3ee78a0531e15be518c7747ba7479f1d41a75 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 16 Oct 2024 13:51:05 +1100 Subject: [PATCH 18/46] Addressing code review comments #994 --- grails-app/conf/data/mapping.json | 7 ------- grails-app/domain/au/org/ala/ecodata/Organisation.groovy | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/grails-app/conf/data/mapping.json b/grails-app/conf/data/mapping.json index 26fcf577a..83398842f 100644 --- a/grails-app/conf/data/mapping.json +++ b/grails-app/conf/data/mapping.json @@ -77,9 +77,6 @@ "organisationFacet": { "type": "keyword" }, - "linkedOrganisationFacet": { - "type": "keyword" - }, "organisationSort": { "type": "keyword", "normalizer" : "case_insensitive_sort" @@ -104,10 +101,6 @@ "organisationId": { "type": "keyword", "copy_to": ["organisationId"] - }, - "organisationName": { - "type": "keyword", - "copy_to": ["organisationName"] } } }, diff --git a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy index 635282c83..6b41a63ee 100644 --- a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy @@ -73,6 +73,7 @@ class Organisation { url nullable: true config nullable: true sourceSystem nullable: true + externalIds nullable: true hubId nullable: true, validator: { String hubId, Organisation organisation, Errors errors -> GormMongoUtil.validateWriteOnceProperty(organisation, 'organisationId', 'hubId', errors) } From c300e2d21f371274824073bedecaf532b71502d0 Mon Sep 17 00:00:00 2001 From: chrisala Date: Fri, 18 Oct 2024 14:20:32 +1100 Subject: [PATCH 19/46] Moved org script to release #994 --- scripts/releases/{5.0 => 5.1}/updateAssociatedOrgNames.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/releases/{5.0 => 5.1}/updateAssociatedOrgNames.js (100%) diff --git a/scripts/releases/5.0/updateAssociatedOrgNames.js b/scripts/releases/5.1/updateAssociatedOrgNames.js similarity index 100% rename from scripts/releases/5.0/updateAssociatedOrgNames.js rename to scripts/releases/5.1/updateAssociatedOrgNames.js From b20081d91d79156cc0b97cbe85b183ed781b357a Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 24 Oct 2024 16:26:51 +1100 Subject: [PATCH 20/46] AtlasOfLivingAustralia/fieldcapture#2789 - shape file upload and shape conversion to GeoJson --- .../org/ala/ecodata/SpatialController.groovy | 120 +++++++++++++++++ .../au/org/ala/ecodata/UrlMappings.groovy | 3 + .../SpatialControllerIntegrationSpec.groovy | 74 ++++++++++ .../resources/projectExtent.zip | Bin 0 -> 2289 bytes .../au/org/ala/ecodata/GeometryUtils.groovy | 20 ++- .../spatial/SpatialConversionUtils.groovy | 112 +++++++++++++++ .../ala/ecodata/spatial/SpatialUtils.groovy | 127 ++++++++++++++++++ 7 files changed, 449 insertions(+), 7 deletions(-) create mode 100644 grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy create mode 100644 src/integration-test/groovy/au/org/ala/ecodata/SpatialControllerIntegrationSpec.groovy create mode 100644 src/integration-test/resources/projectExtent.zip create mode 100644 src/main/groovy/au/org/ala/ecodata/spatial/SpatialConversionUtils.groovy create mode 100755 src/main/groovy/au/org/ala/ecodata/spatial/SpatialUtils.groovy diff --git a/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy b/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy new file mode 100644 index 000000000..972b8ca18 --- /dev/null +++ b/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy @@ -0,0 +1,120 @@ +package au.org.ala.ecodata + +import au.org.ala.ecodata.spatial.SpatialConversionUtils +import au.org.ala.ecodata.spatial.SpatialUtils +import org.apache.commons.fileupload.servlet.ServletFileUpload +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.tuple.Pair +import org.locationtech.jts.geom.Geometry +import org.springframework.web.multipart.MultipartFile + +import javax.servlet.http.HttpServletResponse +@au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.readScope"]) +class SpatialController { + + static responseFormats = ['json', 'xml'] + static allowedMethods = [uploadShapeFile: "POST", getShapeFileFeatureGeoJson: "GET"] + + @au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.writeScope"]) + def uploadShapeFile() { + // Use linked hash map to maintain key ordering + Map retMap = new LinkedHashMap() + + File tmpZipFile = File.createTempFile("shpUpload", ".zip") + + if (ServletFileUpload.isMultipartContent(request)) { + // Parse the request + Map items = request.getFileMap() + + if (items.size() == 1) { + MultipartFile fileItem = items.values()[0] + IOUtils.copy(fileItem.getInputStream(), new FileOutputStream(tmpZipFile)) + retMap.putAll(handleZippedShapeFile(tmpZipFile)) + response.setStatus(HttpServletResponse.SC_OK) + } else { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST) + retMap.put("error", "Multiple files sent in request. A single zipped shape file should be supplied.") + } + } + + respond retMap + } + + @au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.writeScope"]) + def getShapeFileFeatureGeoJson() { + Map retMap + String shapeId = params.shapeFileId + String featureIndex = params.featureId + if (featureIndex != null && shapeId != null) { + + retMap = processShapeFileFeatureRequest(shapeId, featureIndex) + if(retMap.geoJson == null) { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST) + } + else { + response.setStatus(HttpServletResponse.SC_OK) + } + } + else { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST) + retMap = ["error": "featureId and shapeFileId must be supplied"] + } + + respond retMap + } + + private Map processShapeFileFeatureRequest(String shapeFileId, String featureIndex) { + Map retMap = new HashMap() + + try { + File shpFileDir = new File(System.getProperty("java.io.tmpdir"), shapeFileId) + Geometry geoJson = SpatialUtils.getShapeFileFeaturesAsGeometry(shpFileDir, featureIndex) + + if (geoJson == null) { + retMap.put("error", "Invalid geometry") + return retMap + } + else { + if (geoJson.getCoordinates().flatten().size() > grailsApplication.config.getProperty("shapefile.simplify.threshhold", Integer, 50_000)) { + geoJson = GeometryUtils.simplify(geoJson, grailsApplication.config.getProperty("shapefile.simplify.tolerance", Double, 0.0001)) + } + + retMap.put("geoJson", GeometryUtils.geometryToGeoJsonMap(geoJson, grailsApplication.config.getProperty("shapefile.geojson.decimal", Integer, 20))) + } + } catch (Exception ex) { + log.error("Error processsing shapefile feature request", ex) + retMap.put("error", ex.getMessage()) + } + + return retMap + } + + private static Map handleZippedShapeFile(File zippedShp) throws IOException { + // Use linked hash map to maintain key ordering + Map retMap = new LinkedHashMap() + + Pair idFilePair = SpatialConversionUtils.extractZippedShapeFile(zippedShp) + String uploadedShpId = idFilePair.getLeft() + File shpFile = idFilePair.getRight() + + retMap.put("shp_id", uploadedShpId) + + List>> manifestData = SpatialConversionUtils.getShapeFileManifest(shpFile) + + int featureIndex = 0 + for (List> featureData : manifestData) { + // Use linked hash map to maintain key ordering + Map featureDataMap = new LinkedHashMap() + + for (Pair fieldData : featureData) { + featureDataMap.put(fieldData.getLeft(), fieldData.getRight()) + } + + retMap.put(featureIndex, featureDataMap) + + featureIndex++ + } + + return retMap + } +} diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index 4e5d91d5d..45a44e89c 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -42,6 +42,9 @@ class UrlMappings { "/ws/output/getOutputSpeciesUUID/"(controller: "output"){ action = [GET:"getOutputSpeciesUUID"] } + "/ws/shapefile" (controller: "spatial"){ action = [POST:"uploadShapeFile"] } + "/ws/shapefile/geojson/$shapeFileId/$featureId"(controller: "spatial"){ action = [GET:"getShapeFileFeatureGeoJson"] } + "/ws/activitiesForProject/$id" { controller = 'activity' action = 'activitiesForProject' diff --git a/src/integration-test/groovy/au/org/ala/ecodata/SpatialControllerIntegrationSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/SpatialControllerIntegrationSpec.groovy new file mode 100644 index 000000000..1e17d86a7 --- /dev/null +++ b/src/integration-test/groovy/au/org/ala/ecodata/SpatialControllerIntegrationSpec.groovy @@ -0,0 +1,74 @@ +package au.org.ala.ecodata + +import grails.testing.mixin.integration.Integration +import grails.util.GrailsWebMockUtil +import groovy.json.JsonSlurper +import org.apache.http.HttpStatus +import org.grails.plugins.testing.GrailsMockHttpServletRequest +import org.grails.plugins.testing.GrailsMockHttpServletResponse +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.mock.web.MockMultipartFile +import org.springframework.web.context.WebApplicationContext +import spock.lang.Specification + +@Integration +class SpatialControllerIntegrationSpec extends Specification { + + @Autowired + SpatialController spatialController + + @Autowired + WebApplicationContext ctx + + def setup() { + setRequestResponse() + } + + def cleanup() { + } + + def setRequestResponse() { + GrailsMockHttpServletRequest grailsMockHttpServletRequest = new GrailsMockHttpServletRequest() + GrailsMockHttpServletResponse grailsMockHttpServletResponse = new GrailsMockHttpServletResponse() + GrailsWebMockUtil.bindMockWebRequest(ctx, grailsMockHttpServletRequest, grailsMockHttpServletResponse) + } + + void "test uploadShapeFile with resource zip file"() { + given: + // Read the zip file from resources + def zipFileResourceStream = spatialController.class.getResourceAsStream("/projectExtent.zip") + byte[] zipFileBytes = zipFileResourceStream.bytes + + // Mock the request + MockMultipartFile mockMultipartFile = new MockMultipartFile("file", "projectExtent.zip", "application/zip", zipFileBytes) + spatialController.request.addFile(mockMultipartFile) + spatialController.request.method = 'POST' + + when: + // Call the method + spatialController.uploadShapeFile() + + then: + // Verify the response + spatialController.response.status == HttpStatus.SC_OK + println spatialController.response.contentAsString + def responseContent = new JsonSlurper().parseText(spatialController.response.contentAsString) + responseContent.shp_id != null + responseContent["0"].siteId == "340cfe6a-f230-4bb9-a034-23e9bff125c7" + responseContent["0"].name == "Project area for Southern Tablelands Koala Habitat Restoration Project" + + when: + setRequestResponse() + spatialController.request.method = 'GET' + spatialController.params.shapeFileId = responseContent.shp_id + spatialController.params.featureId = "0" + spatialController.getShapeFileFeatureGeoJson() + + then: + spatialController.response.status == HttpStatus.SC_OK + println spatialController.response.contentAsString + def responseJSON = new JsonSlurper().parseText(spatialController.response.contentAsString) + responseJSON.geoJson != null + responseJSON.geoJson.type == "MultiPolygon" + } +} \ No newline at end of file diff --git a/src/integration-test/resources/projectExtent.zip b/src/integration-test/resources/projectExtent.zip new file mode 100644 index 0000000000000000000000000000000000000000..52e846f76af1e9ae1ffb1dfbebf5d9611cff79e8 GIT binary patch literal 2289 zcmb7_c{r4P7sp3NGj@fn*_Vgsk%_EXLXjE!*vmGS&@h%T!dSADFrMhKg=|q-^N>Az zge+x*7#hZ!ow3BcaU%XiDShE{_N1_oR#U7d1PK_Y=Yv!Ilv{|X(aD0N<8TfdWD*2 zscve@_Vcz^*$C~7(A2KZ_ioy8uj~>t6?4ygbpKqi`<1ImfBu~;6%E~8*Wyea#WA7G zXCepk`p(}XxTIzV#|vapk@iQR-LcYvDmzScpyzo}!`MpUgrC=}%{Zsa@Cn;TkgPW$ zCCxXHywb+A0evQsOi{tEqXf@VX`CIXaLxaE3hh9;9+Xb2r#I&AK)6-%u1-LAC+{GQ zxI|z;8w?Xgogi*v{O)ZA>#b#uEYiG9`Bwcw-Lzk7e;CMOWfJH4Caa zL@DNCBOyzc$2_NO#-vqUWi}~ViLfIqn$ADgxu^%3d-MT~G1!&@HPS4?J|Env`#fEz zT_iE8%vCZ-V<1FGgjB*H&bwnm$EO=yXa$kE&? z+)Bz)*F;I?hzPvZQB_SKY-bfXEvC%p6X4(Q@J|>!seoETNa0SJYi2wWu7bi0B*(jdFtP|$R8LN!g#;z3ljPI? z-hb_R;1DW`j$i4_W<{$>U@hAAD)IVvoD@G0M_{Ha@8bV#`OdZBwwH7aZgqz?_SZPzkBdznkq;j2+uE3l5d6bHk9?EgAUC31q>d*lm=< zq52WE5ex)wt*yAF3K+8OaNe8O3*nj2l?vaiKUpeqSSMD+339-_U*G8qMWK&xd{)*v-a1 z!!A&)>}!ST$BkD~)uJ>riN*_dm-Mq?-Yr^YS>y(%eOcNxZuYLMfAsrNY$;s%59xxC zsYljbJ*9Cq%K|&)h<&+W6=;}-MYyA&w;{vV+DSH&;O#&@`hV z+WkBy!Yn(PWibgt%$RED6HVX;N`b)Jm$4h>EFVip7~UiAz2GZCxKtdjjusSVpe+IupP)T*{$0s;qhmNu^xG3j0r1kP~_eijzkASd6 zlVj0l0=XH!D<)&9^=s4$5{WjEOstxMaYq<&$-0G=hW>*Ful%A}4bqYE;$NfQO(bgU z+^av;!MUDvgSc4o?O3-ym4YneQD1%6luMRrB*&PZgYqIO`Tj+mEm!E ze#Cjd97<|me=j{>SG1$O(Hx?gsn-@F3n|xBYVgoy%9s&0=$3k*449D*9#5W0yt?-V z-&e@XJT-u2h~wl;y-cYf2wXnzXCy}nhx~6ys&j0B|E=khgLJ&6Putn0Z3H__pVa&` zjiwCeDVj2RXX%qlpQh<4V||LIbjexzWFb$}@Ut|fH_p;0YjT>F0iB6WDZLSlhT; extractZippedShapeFile(File zippedShpFile) throws IOException { + + File tempDir = Files.createTempDir() + + // Unpack the zipped shape file into the temp directory + ZipFile zf = null + File shpFile = null + try { + zf = new ZipFile(zippedShpFile) + + boolean shpPresent = false + boolean shxPresent = false + boolean dbfPresent = false + + Enumeration entries = zf.entries() + + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement() + InputStream inStream = zf.getInputStream(entry) + File f = new File(tempDir, entry.getName()) + if (!f.getName().startsWith(".")) { + if (entry.isDirectory()) { + f.mkdirs() + } else { + FileOutputStream outStream = new FileOutputStream(f) + IOUtils.copy(inStream, outStream) + + if (entry.getName().endsWith(".shp")) { + shpPresent = true + shpFile = f + } else if (entry.getName().endsWith(".shx") && !f.getName().startsWith("/")) { + shxPresent = true + } else if (entry.getName().endsWith(".dbf") && !f.getName().startsWith("/")) { + dbfPresent = true + } + } + } + } + + if (!shpPresent || !shxPresent || !dbfPresent) { + throw new IllegalArgumentException("Invalid archive. Must contain .shp, .shx and .dbf at a minimum.") + } + } catch (Exception e) { + log.error(e.getMessage(), e) + } finally { + if (zf != null) { + try { + zf.close() + } catch (Exception e) { + log.error(e.getMessage(), e) + } + } + } + + if (shpFile == null) { + return null + } else { + return Pair.of(shpFile.getParentFile().getName(), shpFile) + } + } + + static List>> getShapeFileManifest(File shpFile) throws IOException { + List>> manifestData = new ArrayList>>() + + FileDataStore store = FileDataStoreFinder.getDataStore(shpFile) + + SimpleFeatureSource featureSource = store.getFeatureSource(store.getTypeNames()[0]) + SimpleFeatureCollection featureCollection = featureSource.getFeatures() + SimpleFeatureIterator it = featureCollection.features() + + while (it.hasNext()) { + SimpleFeature feature = it.next() + List> pairList = new ArrayList>() + for (Property prop : feature.getProperties()) { + if (!(prop.getType() instanceof GeometryType)) { + Pair pair = Pair.of(prop.getName().toString(), feature.getAttribute(prop.getName())) + pairList.add(pair) + } + } + manifestData.add(pairList) + } + + return manifestData + } +} + diff --git a/src/main/groovy/au/org/ala/ecodata/spatial/SpatialUtils.groovy b/src/main/groovy/au/org/ala/ecodata/spatial/SpatialUtils.groovy new file mode 100755 index 000000000..948b2290a --- /dev/null +++ b/src/main/groovy/au/org/ala/ecodata/spatial/SpatialUtils.groovy @@ -0,0 +1,127 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package au.org.ala.ecodata.spatial + + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.geotools.data.FileDataStore +import org.geotools.data.FileDataStoreFinder +import org.geotools.data.simple.SimpleFeatureCollection +import org.geotools.data.simple.SimpleFeatureIterator +import org.geotools.data.simple.SimpleFeatureSource +import org.geotools.geometry.jts.JTS +import org.geotools.geometry.jts.JTSFactoryFinder +import org.geotools.referencing.CRS +import org.geotools.referencing.crs.DefaultGeographicCRS +import org.locationtech.jts.geom.Geometry +import org.locationtech.jts.geom.GeometryCollection +import org.locationtech.jts.geom.GeometryFactory +import org.opengis.feature.simple.SimpleFeature +import org.opengis.referencing.crs.CoordinateReferenceSystem + +@CompileStatic +@Slf4j +class SpatialUtils { + static Geometry getShapeFileFeaturesAsGeometry(File shpFileDir, String featureIndexes) throws IOException { + + if (!shpFileDir.exists() || !shpFileDir.isDirectory()) { + throw new IllegalArgumentException("Supplied directory does not exist or is not a directory") + } + + List geometries = new ArrayList() + FileDataStore store = null + SimpleFeatureIterator it = null + + try { + + File shpFile = null + for (File f : shpFileDir.listFiles()) { + if (f.getName().endsWith(".shp")) { + shpFile = f + break + } + } + + if (shpFile == null) { + throw new IllegalArgumentException("No .shp file present in directory") + } + + store = FileDataStoreFinder.getDataStore(shpFile) + + SimpleFeatureSource featureSource = store.getFeatureSource(store.getTypeNames()[0]) + SimpleFeatureCollection featureCollection = featureSource.getFeatures() + it = featureCollection.features() + + //transform CRS to the same as the shapefile (at least try) + //default to 4326 + CoordinateReferenceSystem crs = null + try { + crs = store.getSchema().getCoordinateReferenceSystem() + if (crs == null) { + //attempt to parse prj + try { + File prjFile = new File(shpFile.getPath().substring(0, shpFile.getPath().length() - 3) + "prj") + if (prjFile.exists()) { + String prj = prjFile.text + + if (prj == "PROJCS[\"WGS_1984_Web_Mercator_Auxiliary_Sphere\",GEOGCS[\"GCS_WGS_1984\",DATUM[\"D_WGS_1984\",SPHEROID[\"WGS_1984\",6378137.0,298.257223563]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]],PROJECTION[\"Mercator_Auxiliary_Sphere\"],PARAMETER[\"False_Easting\",0.0],PARAMETER[\"False_Northing\",0.0],PARAMETER[\"Central_Meridian\",0.0],PARAMETER[\"Standard_Parallel_1\",0.0],PARAMETER[\"Auxiliary_Sphere_Type\",0.0],UNIT[\"Meter\",1.0]]") { + //support for arcgis online default shp exports + crs = CRS.decode("EPSG:3857") + } else { + crs = CRS.parseWKT(prjFile.text) + } + } + } catch (Exception ignored) { + } + + if (crs == null) { + crs = DefaultGeographicCRS.WGS84 + } + } + } catch (Exception ignored) { + } + + int i = 0 + boolean all = "all".equalsIgnoreCase(featureIndexes) + def indexes = [] + if (!all) featureIndexes.split(",").each { indexes.push(it.toInteger()) } + while (it.hasNext()) { + SimpleFeature feature = (SimpleFeature) it.next() + if (all || indexes.contains(i)) { + geometries.add(feature.getDefaultGeometry() as Geometry) + } + i++ + } + + Geometry mergedGeometry + + if (geometries.size() == 1) { + mergedGeometry = geometries.get(0) + } else { + GeometryFactory factory = JTSFactoryFinder.getGeometryFactory(null) + GeometryCollection geometryCollection = (GeometryCollection) factory.buildGeometry(geometries) + + // note the following geometry collection may be invalid (say with overlapping polygons) + mergedGeometry = geometryCollection.union() + } + + try { + return JTS.transform(mergedGeometry, CRS.findMathTransform(crs, DefaultGeographicCRS.WGS84, true)) + } catch (Exception ignored) { + return mergedGeometry + } + } catch (Exception e) { + throw e + } finally { + if (it != null) { + it.close() + } + if (store != null) { + store.dispose() + } + } + } +} From 8270543768236113dcbcc9a89f39f8878c39ade1 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 31 Oct 2024 16:40:55 +1100 Subject: [PATCH 21/46] AtlasOfLivingAustralia/fieldcapture#1724 - manual project geographic information ui support - spatial object name standardisation --- grails-app/conf/application.groovy | 19 ++- .../org/ala/ecodata/ProjectController.groovy | 9 ++ .../org/ala/ecodata/SpatialController.groovy | 15 +- .../au/org/ala/ecodata/UrlMappings.groovy | 1 + .../au/org/ala/ecodata/GeographicInfo.groovy | 3 + .../au/org/ala/ecodata/ProjectService.groovy | 50 +++++- .../au/org/ala/ecodata/SpatialService.groovy | 81 +++++++++- .../5.1/updateSiteLocationMetadata.js | 143 ++++++++++++++++++ .../reporting/ProjectXlsExporter.groovy | 47 +----- .../org/ala/ecodata/ProjectServiceSpec.groovy | 75 +++++++++ .../ala/ecodata/SearchControllerSpec.groovy | 2 +- .../org/ala/ecodata/SpatialServiceSpec.groovy | 6 +- .../reporting/ProjectXlsExporterSpec.groovy | 2 +- 13 files changed, 399 insertions(+), 54 deletions(-) create mode 100644 scripts/releases/5.1/updateSiteLocationMetadata.js diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index c7e1fa5c2..41b405694 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -496,7 +496,24 @@ app { } checkForBoundaryIntersectionInLayers = [ "cl927", "cl11163" ] } - displayNames = [elect: "Electorate(s)", state: "State(s)"] + displayNames = [ + elect: [ + headerName: "Electorate(s)" + ], + state: [ + headerName: "State(s)", + mappings: [ + "Northern Territory": ["Northern Territory (including Coastal Waters)", "NT"], + "Tasmania": ["Tasmania (including Coastal Waters)", "TAS"], + "New South Wales": ["New South Wales (including Coastal Waters)", "NSW"], + "Victoria": ["Victoria (including Coastal Waters)", "VIC"], + "Queensland": ["Queensland (including Coastal Waters)", "QLD"], + "South Australia": ["South Australia (including Coastal Waters)", "SA"], + "Australian Capital Territory": ["ACT"], + "Western Australia": ["Western Australia (including Coastal Waters)", "WA"], + ] + ] + ] } } /******************************************************************************\ diff --git a/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy b/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy index 3b6680a33..22c77d8c2 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy @@ -324,6 +324,15 @@ class ProjectController { render result as JSON } + def findStateAndElectorateForProject() { + if (!params.projectId) { + render status:400, text: "projectId is a required parameter" + } else { + Map project = projectService.get(params.projectId) + asJson projectService.findStateAndElectorateForProject(project) + } + } + def findByName() { if (!params.projectName) { render status:400, text: "projectName is a required parameter" diff --git a/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy b/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy index 972b8ca18..e3d8aa7af 100644 --- a/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy @@ -11,7 +11,7 @@ import org.springframework.web.multipart.MultipartFile import javax.servlet.http.HttpServletResponse @au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.readScope"]) class SpatialController { - + SpatialService spatialService static responseFormats = ['json', 'xml'] static allowedMethods = [uploadShapeFile: "POST", getShapeFileFeatureGeoJson: "GET"] @@ -63,6 +63,19 @@ class SpatialController { respond retMap } + def features() { + def retVariable + if (!params.layerId) { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST) + retVariable = ["error": "layerId must be supplied"] + } + else { + retVariable = spatialService.features(params.layerId) + } + + respond retVariable + } + private Map processShapeFileFeatureRequest(String shapeFileId, String featureIndex) { Map retMap = new HashMap() diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index 45a44e89c..0c93fd480 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -198,6 +198,7 @@ class UrlMappings { "/ws/project/getBiocollectFacets"(controller: "project"){ action = [GET:"getBiocollectFacets"] } "/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets") "/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords") + "/ws/project/findStateAndElectorateForProject"(controller: "project", action: "findStateAndElectorateForProject") "/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch") "/ws/dataSetSummary/$projectId/$dataSetId?"(controller :'dataSetSummary') { diff --git a/grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy b/grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy index e12cef264..e2b0afc7a 100644 --- a/grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy +++ b/grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy @@ -20,6 +20,9 @@ class GeographicInfo { /** Some projects don't have specific geographic areas and are flagged as being run nationwide */ boolean nationwide = false + /** A flag to override calculated values for states and electorates with manually entered values */ + boolean isDefault = false + /** The primary state in which this project is running, if applicable */ String primaryState diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 84e647b98..33919cd94 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -4,7 +4,6 @@ import au.org.ala.ecodata.converter.SciStarterConverter import grails.converters.JSON import grails.core.GrailsApplication import groovy.json.JsonSlurper -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.context.MessageSource import org.springframework.web.servlet.i18n.SessionLocaleResolver @@ -1249,6 +1248,55 @@ class ProjectService { [] } + /** + * Find primary/other state(s)/electorate(s) for a project. + * 1. If isDefault is true, use manually assigned state(s)/electorate(s) i.e project.geographicInfo. + * 2. If isDefault is false or missing, use the state(s)/electorate(s) from sites using site precedence. + * 3. If isDefault is false and there are no sites, use manual state(s)/electorate(s) in project.geographicInfo. + */ + Map findStateAndElectorateForProject(Map project) { + Map result = [:] + if(project == null) { + return result + } + + Map geographicInfo = project?.geographicInfo + // isDefault is false or missing + if (geographicInfo == null || (geographicInfo.isDefault == false)) { + Map intersections = orderLayerIntersectionsByAreaOfProjectSites(project) + Map config = metadataService.getGeographicConfig() + List intersectionLayers = config.checkForBoundaryIntersectionInLayers + intersectionLayers?.each { layer -> + Map facetName = metadataService.getGeographicFacetConfig(layer) + if (facetName.name) { + List intersectionValues = intersections[layer] + if (intersectionValues) { + result["primary${facetName.name}"] = intersectionValues.pop() + result["other${facetName.name}"] = intersectionValues.join("; ") + } + } + else + log.error ("No facet config found for layer $layer.") + } + } + + //isDefault is true or false and no sites. + if (geographicInfo) { + // load from manually assigned electorates/states + if (!result.containsKey("primaryelect")) { + result["primaryelect"] = geographicInfo.primaryElectorate + result["otherelect"] = geographicInfo.otherElectorates?.join("; ") + } + + if (!result.containsKey("primarystate")) { + result["primarystate"] = geographicInfo.primaryState + result["otherstate"] = geographicInfo.otherStates?.join("; ") + } + } + + result + } + /** * Returns a distinct list of hubIds for the supplied projects. * @param projects diff --git a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy index e48b1af84..850394284 100644 --- a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy @@ -4,6 +4,7 @@ import grails.core.GrailsApplication import grails.plugin.cache.Cacheable import groovy.json.JsonParserType import groovy.json.JsonSlurper +import org.apache.commons.lang.WordUtils import org.locationtech.jts.geom.* import org.locationtech.jts.io.WKTReader @@ -26,9 +27,11 @@ class SpatialService { WebService webService MetadataService metadataService + CacheService cacheService GrailsApplication grailsApplication Map lookupTable + Map synonymLookupTable = [:] public SpatialService() { JsonSlurper js = new JsonSlurper() @@ -214,10 +217,12 @@ class SpatialService { Map intersectionAreaByFacets = [:].withDefault { [:] } response?.each { String fid, List matchingObjects -> filteredResponse[fid] = [] + Map facetConfig = metadataService.getGeographicFacetConfig(fid) // check for boundary intersection object for selected layers defined in config. if (checkForBoundaryIntersectionInLayers.contains(fid)) { matchingObjects.each { Map obj -> String boundaryPid = obj.pid + String objName = obj.name = standardiseSpatialLayerObjectName(obj.name, facetConfig.name) if (boundaryPid) { log.debug("Intersecting ${obj.fieldname}(${fid}) - ${obj.name} ") // Get geoJSON of the object stored in spatial portal @@ -241,9 +246,9 @@ class SpatialService { if (isValidGeometryIntersection(mainGeometry, boundaryGeometry)) { filteredResponse[fid].add(obj) def (intersectionAreaOfMainGeometry, area) = getIntersectionProportionAndArea(mainGeometry, boundaryGeometry) - intersectionAreaByFacets[fid][obj.name] = area + intersectionAreaByFacets[fid][objName] = area } else { - log.debug("Filtered out ${obj.fieldname}(${fid}) - ${obj.name}") + log.debug("Filtered out ${obj.fieldname}(${fid}) - ${objName}") } end = System.currentTimeMillis() @@ -377,6 +382,74 @@ class SpatialService { GeometryUtils.wktToGeoJson(resp) } + /** + * Fetch spatial layer objects and standardise object names. + * @param layerId + * @return + */ + List features (String layerId) { + cacheService.get("features-${layerId}", { + def resp = webService.getJson("${grailsApplication.config.getProperty('spatial.baseUrl')}/ws/objects/${layerId}") + Map facetName = null + try { + facetName = metadataService.getGeographicFacetConfig(layerId) + if(resp instanceof List) { + return resp.collect { obj -> + obj.name = standardiseSpatialLayerObjectName(obj.name, facetName.name) + obj + } + } + } + catch (IllegalArgumentException e) { + log.error("Error getting facet config for layer $layerId") + } + + return [] + }, 365) as List + } + + /** + * Get mapping for a facet from config + * @param facetName + * @return + */ + Map getDisplayNamesForFacet(String facetName) { + Map lookupTable = grailsApplication.config.getProperty('app.facets.displayNames', Map) + if (facetName) { + return lookupTable[facetName]?.mappings ?: [:] + } + } + + /** + * Spatial portal returns the object name in a variety of formats. This function formats the object name to a more + * consistent way. For example, depending on layer used New South Wales is sometimes called "New South Wales (including Coastal Waters)". + * @param name - name of the object + * @param synonymTable - expected data format - ["New South Wales": ["New South Wales (including Coastal Waters)", "NSW"], "Australian Capital Territory": ["ACT"]] + * @return + */ + String standardiseSpatialLayerObjectName(String name, Map synonymTable, String facetName) { + if (name) { + name = name.trim().toLowerCase() + // initialise a Map that stores the inverse of mappings. ["act": "Australian Capital Territory", "nsw": "New South Wales"] + if (synonymLookupTable[facetName] == null) { + synonymLookupTable[facetName] = synonymTable?.collectEntries { k, List v -> v.collectEntries { v1 -> [(v1.toLowerCase()): k] } } + } + + synonymLookupTable[facetName]?.get(name) ?: WordUtils.capitalize(name) + } + } + + /** + * Provide a facet name such as "state", "elect" etc. to get standardised object name. + * @param name - object name such as "New South Wales (including Coastal Waters)" + * @param facetName - facet name such as "state", "elect" + * @return + */ + String standardiseSpatialLayerObjectName(String name, String facetName) { + Map lookupTable = getDisplayNamesForFacet(facetName) + standardiseSpatialLayerObjectName(name, lookupTable, facetName) + } + /** * Converts the response from the spatial portal into geographic facets, taking into account the facet * configuration (whether the facet is made up of a single layer or a group of layers). @@ -393,11 +466,11 @@ class SpatialService { // Grouped facets combine multiple layers into a single facet. If the site intersects with // any object in the layer, then that layer is added as a matching value to the facet. if (matchingObjects) { - result[facetConfig.name].add(matchingObjects[0].fieldname) + result[facetConfig.name].add(standardiseSpatialLayerObjectName(matchingObjects[0].fieldname as String, facetConfig.name as String)) } } else { - result[facetConfig.name] = matchingObjects.collect{it.name} + result[facetConfig.name] = matchingObjects.collect{standardiseSpatialLayerObjectName(it.name as String, facetConfig.name as String)} } } result diff --git a/scripts/releases/5.1/updateSiteLocationMetadata.js b/scripts/releases/5.1/updateSiteLocationMetadata.js new file mode 100644 index 000000000..d1f886582 --- /dev/null +++ b/scripts/releases/5.1/updateSiteLocationMetadata.js @@ -0,0 +1,143 @@ +load('../../utils/audit.js'); +const intersection = "intersectionAreaByFacets" +let lookupTable = { + "state": { + "Northern Territory": ["Northern Territory (including Coastal Waters)"], + "Tasmania": ["Tasmania (including Coastal Waters)"], + "New South Wales": ["New South Wales (including Coastal Waters)"], + "Victoria": ["Victoria (including Coastal Waters)"], + "Queensland": ["Queensland (including Coastal Waters)"], + "South Australia": ["South Australia (including Coastal Waters)"], + "Australian Capital Territory": [], + "Western Australia": ["Western Australia (including Coastal Waters)"], + } +} + +const propertiesToStandardize = ["state", "elect"]; +function standardiseSpatialLayerObjectName(name, property) { + if (name) { + let lookupTableForProperty = lookupTable[property]; + name = name.trim().toLowerCase(); + if (lookupTableForProperty) { + let manyToOneMappingTable = `${property}Synonym`; + if (!lookupTable[manyToOneMappingTable]) { + const mappings = {}; + const keys= Object.keys(lookupTableForProperty || {}) + for (let i = 0; i < keys.length; i++) { + let key = keys[i]; + let values = lookupTableForProperty[key]; + values.forEach(value => { + mappings[value.toLowerCase()] = key; + }); + } + + lookupTable[manyToOneMappingTable] = mappings; + } + + if (lookupTable[manyToOneMappingTable][name]) + return lookupTable[manyToOneMappingTable][name]; + } + + return name.replace(/\b\w/g, char => char.toUpperCase()); + } +} + +/** + * + * @param geometry = { + * "state": ["New South Wales"], + * "elect": "Page" + * } + * @param updated + * @returns {*} + */ +function standardiseFacetValues(geometry, updated) { + propertiesToStandardize.forEach(property => { + let value = geometry[property]; + if (value) { + if (typeof value === "string"){ + value = geometry[property] = [value]; + updated = true; + } + + if (value.length > 0) { + value.forEach((item, index) => { + let standardizedValue = standardiseSpatialLayerObjectName(item, property); + if (standardizedValue !== item) { + value[index] = standardizedValue; + updated = true; + } + }); + } + } + }); + + return updated; +} + +/** + * @param geometry = { + * "state": ["New South Wales"], + * "intersectionAreaByFacets": { + * "state": { + * "CURRENT": { + * "New South Wales": 0 + * }, + * "cl927": { + * "New South Wales": 0 + * } + * }, + * "elect": { + * "CURRENT": { + * "Page": 0 + * }, + * "cl11163": { + * "Page": 0 + * } + * } + * } + * } + * @param updated + * @returns updated + */ +function standardiseIntersectionAreaByFacets(geometry, updated) { + let intersectionAreaByFacets = geometry[intersection]; + if (intersectionAreaByFacets) { + var facets = Object.keys(intersectionAreaByFacets); + for (let i = 0; i < facets.length; i++) { + let facet = facets[i]; + let layersSpatialNamesAndArea = intersectionAreaByFacets[facet]; + let layers = Object.keys(layersSpatialNamesAndArea); + for (let j = 0; j < layers.length; j++) { + let layer = layers[j]; + let spatialNamesAndArea = layersSpatialNamesAndArea[layer]; + let spatialNames = Object.keys(spatialNamesAndArea); + let newSpatialValuesAndArea = {}; + for (let k = 0; k < spatialNames.length; k++) { + let newSpatialValue = standardiseSpatialLayerObjectName(spatialNames[k], facet); + newSpatialValuesAndArea[newSpatialValue] = spatialNamesAndArea[spatialNames[k]]; + updated = true; + } + + layersSpatialNamesAndArea[layer] = newSpatialValuesAndArea; + } + } + } + + return updated; +} + +db.site.find({}).forEach(site => { + let updated = false; + let geometry = site.extent && site.extent.geometry; + if (geometry) { + updated = standardiseFacetValues(geometry, updated); + updated = standardiseIntersectionAreaByFacets(geometry, updated); + + if (updated) { + print(`Updating site ${site.siteId}`); + db.site.updateOne({siteId: site.siteId}, {$set: {"extent.geometry": geometry}}); + audit(site, site.siteId, 'au.org.ala.ecodata.Site', 'system'); + } + } +}); \ No newline at end of file diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index a702f2d13..4e323c885 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -1,22 +1,11 @@ package au.org.ala.ecodata.reporting -import au.org.ala.ecodata.ActivityForm -import au.org.ala.ecodata.AssociatedOrg -import au.org.ala.ecodata.DataDescription -import au.org.ala.ecodata.ExternalId -import au.org.ala.ecodata.ManagementUnit -import au.org.ala.ecodata.ManagementUnitService -import au.org.ala.ecodata.ProjectService -import au.org.ala.ecodata.Organisation -import au.org.ala.ecodata.OrganisationService -import au.org.ala.ecodata.Program -import au.org.ala.ecodata.ProgramService +import au.org.ala.ecodata.* import au.org.ala.ecodata.metadata.OutputModelProcessor import grails.util.Holders import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory import pl.touk.excel.export.multisheet.AdditionalSheet - /** * Exports project, site, activity and output data to a Excel spreadsheet. */ @@ -282,12 +271,12 @@ class ProjectXlsExporter extends ProjectExporter { private static String getHeaderNameForFacet (String facetName, String prefix = "Primary") { Map names = Holders.config.getProperty("app.facets.displayNames", Map) - String name = names[facetName] + String name = names[facetName]['headerName'] return "$prefix $name (Interpreted)" } private static String getPropertyNameForFacet (String facetName, String prefix = "primary") { - return "interpreted_$prefix$facetName" + return "$prefix$facetName" } void export(Map project) { @@ -340,34 +329,8 @@ class ProjectXlsExporter extends ProjectExporter { } private void addPrimaryAndOtherIntersections (Map project) { - Map intersections = projectService.orderLayerIntersectionsByAreaOfProjectSites(project) - Map config = metadataService.getGeographicConfig() - List intersectionLayers = config.checkForBoundaryIntersectionInLayers - intersectionLayers?.each { layer -> - Map facetName = metadataService.getGeographicFacetConfig(layer) - if (facetName.name) { - List intersectionValues = intersections[layer] - if (intersectionValues) { - project[getPropertyNameForFacet(facetName.name)] = intersectionValues.pop() - project[getPropertyNameForFacet(facetName.name,"other")] = intersectionValues.join("; ") - } - } - else - log.error ("No facet config found for layer $layer.") - } - - if (project.geographicInfo) { - // load from manually assigned electorates/states - if (!project.containsKey(getPropertyNameForFacet("elect"))) { - project[getPropertyNameForFacet("elect")] = project.geographicInfo.primaryElectorate - project[getPropertyNameForFacet("elect","other")] = project.geographicInfo.otherElectorates?.join("; ") - } - - if (!project.containsKey(getPropertyNameForFacet("state"))) { - project[getPropertyNameForFacet("state")] = project.geographicInfo.primaryState - project[getPropertyNameForFacet("state","other")] = project.geographicInfo.otherStates?.join("; ") - } - } + Map result = projectService.findStateAndElectorateForProject(project) ?: [:] + project << result } private addProjectGeo(Map project) { diff --git a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy index 5f134a0a6..cc0192f5a 100644 --- a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy @@ -1099,4 +1099,79 @@ class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest> geographicConfig + metadataService.getGeographicConfig(*_) >> geographicConfig + metadataService.getGeographicFacetConfig("layer1") >> [name: "state", grouped: false] + metadataService.getGeographicFacetConfig("layer1", _) >> [name: "state", grouped: false] + metadataService.getGeographicFacetConfig("layer2") >> [name: "elect", grouped: false] + metadataService.getGeographicFacetConfig("layer2", _) >> [name: "elect", grouped: false] + + when: + Map result = service.findStateAndElectorateForProject(project) + + then: + result.primarystate == "state1" + result.otherstate == "state2; state3" + result.primaryelect == "electorate2" + result.otherelect == "electorate1" + } + + def "findStateAndElectorateForProject should return default geographic info if isDefault is false and project sites are empty"() { + given: + Map project = [geographicInfo: [isDefault: false, primaryState: "ACT", otherStates: ['NSW', 'VIC'], primaryElectorate: "Bean", otherElectorates: ['Canberra', 'Fenner']]] + Map geographicConfig = [ + contextual: [state: 'layer1', elect: 'layer2'], + checkForBoundaryIntersectionInLayers: ["layer1", "layer2"] + ] + + metadataService.getGeographicConfig(*_) >> geographicConfig + metadataService.getGeographicFacetConfig("layer1") >> [name: "state", grouped: false] + metadataService.getGeographicFacetConfig("layer2") >> [name: "elect", grouped: false] + service.getRepresentativeSitesOfProject(project) >> [] + + + when: + Map result = service.findStateAndElectorateForProject(project) + + then: + result.primarystate == "ACT" + result.otherstate == "NSW; VIC" + result.primaryelect == "Bean" + result.otherelect == "Canberra; Fenner" + } + + + def "findStateAndElectorateForProject should return default geographic info if isDefault is true"() { + given: + Map project = [geographicInfo: [isDefault: true, primaryState: "ACT", otherStates: ['NSW', 'VIC'], primaryElectorate: "Bean", otherElectorates: ['Canberra', 'Fenner']]] + + when: + Map result = service.findStateAndElectorateForProject(project) + + then: + result.primarystate == "ACT" + result.otherstate == "NSW; VIC" + result.primaryelect == "Bean" + result.otherelect == "Canberra; Fenner" + } + + def "findStateAndElectorateForProject should return empty map if project is null"() { + when: + Map project = null + Map result = service.findStateAndElectorateForProject(project) + + then: + result.isEmpty() + } } diff --git a/src/test/groovy/au/org/ala/ecodata/SearchControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SearchControllerSpec.groovy index 67a08bcc1..8a587509f 100644 --- a/src/test/groovy/au/org/ala/ecodata/SearchControllerSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SearchControllerSpec.groovy @@ -90,7 +90,7 @@ class SearchControllerSpec extends Specification implements ControllerUnitTest> getShape2() 1 * webService.get("/ws/shapes/wkt/456") >> getBoundaryShape() } diff --git a/src/test/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporterSpec.groovy index b09cf2984..aa64f203a 100644 --- a/src/test/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporterSpec.groovy @@ -230,7 +230,7 @@ class ProjectXlsExporterSpec extends Specification implements GrailsUnitTest { setup: String sheet = "Electorate Coord" Map project = project() - projectService.orderLayerIntersectionsByAreaOfProjectSites(_) >> ["cl927": ["ACT"], "cl11163": ["bean", "fenner", "canberra"]] + projectService.findStateAndElectorateForProject(_) >> ["primarystate": "ACT", "otherstate": null, "primaryelect": "bean", "otherelect": "fenner; canberra"] when: projectXlsExporter.export(project) From 4f7fa4ae1d3c6c970587abcb4ac2a93c5b8c6aa2 Mon Sep 17 00:00:00 2001 From: chrisala Date: Fri, 1 Nov 2024 13:26:31 +1100 Subject: [PATCH 22/46] Add external id for tech one orgs #994 --- grails-app/domain/au/org/ala/ecodata/ExternalId.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy b/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy index 8ab58aed6..175387554 100644 --- a/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy +++ b/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy @@ -12,7 +12,7 @@ class ExternalId implements Comparable { enum IdType { INTERNAL_ORDER_NUMBER, TECH_ONE_CODE, WORK_ORDER, GRANT_AWARD, GRANT_OPPORTUNITY, RELATED_PROJECT, - MONITOR_PROTOCOL_INTERNAL_ID, MONITOR_PROTOCOL_GUID, TECH_ONE_CONTRACT_NUMBER, MONITOR_PLOT_GUID, + MONITOR_PROTOCOL_INTERNAL_ID, MONITOR_PROTOCOL_GUID, TECH_ONE_CONTRACT_NUMBER, TECH_ONE_PARTY_ID, MONITOR_PLOT_GUID, MONITOR_PLOT_SELECTION_GUID, MONITOR_MINTED_COLLECTION_ID, UNSPECIFIED } static constraints = { From be9439f89be2cb9bc409f6c02923d01782941b40 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 1 Nov 2024 16:47:42 +1100 Subject: [PATCH 23/46] AtlasOfLivingAustralia/fieldcapture#1724 & #1034 - added intersectWith to find states of an electorate - added statewide field - updated script to standardise project geographicInfo fields --- grails-app/conf/application.groovy | 2 +- .../org/ala/ecodata/SpatialController.groovy | 3 +- .../au/org/ala/ecodata/GeographicInfo.groovy | 3 ++ .../au/org/ala/ecodata/SpatialService.groovy | 20 +++++-- .../5.1/updateSiteLocationMetadata.js | 53 +++++++++++++++---- 5 files changed, 66 insertions(+), 15 deletions(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 41b405694..81bf9480f 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -510,7 +510,7 @@ app { "Queensland": ["Queensland (including Coastal Waters)", "QLD"], "South Australia": ["South Australia (including Coastal Waters)", "SA"], "Australian Capital Territory": ["ACT"], - "Western Australia": ["Western Australia (including Coastal Waters)", "WA"], + "Western Australia": ["Western Australia (including Coastal Waters)", "WA"] ] ] ] diff --git a/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy b/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy index e3d8aa7af..39fe0c520 100644 --- a/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy @@ -70,7 +70,8 @@ class SpatialController { retVariable = ["error": "layerId must be supplied"] } else { - retVariable = spatialService.features(params.layerId) + List intersectWith = params.intersectWith?.split(",") ?: [] + retVariable = spatialService.features(params.layerId, intersectWith) } respond retVariable diff --git a/grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy b/grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy index e2b0afc7a..5a14a7a7d 100644 --- a/grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy +++ b/grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy @@ -20,6 +20,9 @@ class GeographicInfo { /** Some projects don't have specific geographic areas and are flagged as being run nationwide */ boolean nationwide = false + /** A flag to indicate that the project is running statewide i.e. all electorates in a state */ + boolean statewide = false + /** A flag to override calculated values for states and electorates with manually entered values */ boolean isDefault = false diff --git a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy index 850394284..ab30d9074 100644 --- a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy @@ -387,17 +387,31 @@ class SpatialService { * @param layerId * @return */ - List features (String layerId) { - cacheService.get("features-${layerId}", { + List features (String layerId, List intersectWith = []) { + cacheService.get("features-${layerId}-intersect-with-${intersectWith.join('')}", { def resp = webService.getJson("${grailsApplication.config.getProperty('spatial.baseUrl')}/ws/objects/${layerId}") Map facetName = null try { facetName = metadataService.getGeographicFacetConfig(layerId) if(resp instanceof List) { - return resp.collect { obj -> + resp.sort { it.name } + List objects = resp.collect { obj -> obj.name = standardiseSpatialLayerObjectName(obj.name, facetName.name) + + intersectWith.each { String fid -> + def intersectedObjects = webService.getJson("${grailsApplication.config.getProperty('spatial.baseUrl')}/ws/intersect/object/${fid}/${obj.pid}") + if (intersectedObjects instanceof List) { + Map facetConfig = metadataService.getGeographicFacetConfig(fid) + intersectedObjects.sort { it.name } + obj[(facetConfig.name)] = obj[fid] = intersectedObjects.collect { standardiseSpatialLayerObjectName(it.name, facetConfig.name) } + } + } + obj } + + + return objects } } catch (IllegalArgumentException e) { diff --git a/scripts/releases/5.1/updateSiteLocationMetadata.js b/scripts/releases/5.1/updateSiteLocationMetadata.js index d1f886582..173919696 100644 --- a/scripts/releases/5.1/updateSiteLocationMetadata.js +++ b/scripts/releases/5.1/updateSiteLocationMetadata.js @@ -2,14 +2,14 @@ load('../../utils/audit.js'); const intersection = "intersectionAreaByFacets" let lookupTable = { "state": { - "Northern Territory": ["Northern Territory (including Coastal Waters)"], - "Tasmania": ["Tasmania (including Coastal Waters)"], - "New South Wales": ["New South Wales (including Coastal Waters)"], - "Victoria": ["Victoria (including Coastal Waters)"], - "Queensland": ["Queensland (including Coastal Waters)"], - "South Australia": ["South Australia (including Coastal Waters)"], - "Australian Capital Territory": [], - "Western Australia": ["Western Australia (including Coastal Waters)"], + "Northern Territory": ["Northern Territory (including Coastal Waters)", "NT"], + "Tasmania": ["Tasmania (including Coastal Waters)", "TAS"], + "New South Wales": ["New South Wales (including Coastal Waters)", "NSW"], + "Victoria": ["Victoria (including Coastal Waters)", "VIC"], + "Queensland": ["Queensland (including Coastal Waters)", "QLD"], + "South Australia": ["South Australia (including Coastal Waters)", "SA"], + "Australian Capital Territory": ["ACT"], + "Western Australia": ["Western Australia (including Coastal Waters)", "WA"] } } @@ -40,6 +40,8 @@ function standardiseSpatialLayerObjectName(name, property) { return name.replace(/\b\w/g, char => char.toUpperCase()); } + + return null; } /** @@ -127,7 +129,23 @@ function standardiseIntersectionAreaByFacets(geometry, updated) { return updated; } -db.site.find({}).forEach(site => { +function standardiseGeographicInfo(geographicInfo) { + if (geographicInfo) { + geographicInfo.primaryState = standardiseSpatialLayerObjectName(geographicInfo.primaryState, "state"); + if (geographicInfo.otherStates) { + geographicInfo.otherStates = geographicInfo.otherStates.map(state => standardiseSpatialLayerObjectName(state, "state")); + } + + geographicInfo.primaryElectorate = standardiseSpatialLayerObjectName(geographicInfo.primaryElectorate, "elect"); + if (geographicInfo.otherElectorates) { + geographicInfo.otherElectorates = geographicInfo.otherElectorates.map(elect => standardiseSpatialLayerObjectName(elect, "elect")); + } + } + + return geographicInfo; +} + +db.site.find({"site.extent.geometry": {$exists: true}}).forEach(site => { let updated = false; let geometry = site.extent && site.extent.geometry; if (geometry) { @@ -140,4 +158,19 @@ db.site.find({}).forEach(site => { audit(site, site.siteId, 'au.org.ala.ecodata.Site', 'system'); } } -}); \ No newline at end of file +}); + +print("Completed sites"); + +db.project.find({ + "geographicInfo": {$exists: true} +}).forEach(project => { + print(`Updating project ${project.projectId}`); + if(project.geographicInfo) { + project.geographicInfo = standardiseGeographicInfo(project.geographicInfo); + db.project.updateOne({projectId: project.projectId}, {$set: {"geographicInfo": project.geographicInfo}}); + audit(project, project.projectId, 'au.org.ala.ecodata.Project', 'system'); + } +}); + +print("Completed projects"); \ No newline at end of file From 8c314f01aed5ca9b1999e1cd78c4ea83cc5bb7a2 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 5 Nov 2024 13:04:53 +1100 Subject: [PATCH 24/46] AtlasOfLivingAustralia/fieldcapture#1724 & #1034 - fixed an error with query --- scripts/releases/5.1/updateSiteLocationMetadata.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/releases/5.1/updateSiteLocationMetadata.js b/scripts/releases/5.1/updateSiteLocationMetadata.js index 173919696..dc7f98424 100644 --- a/scripts/releases/5.1/updateSiteLocationMetadata.js +++ b/scripts/releases/5.1/updateSiteLocationMetadata.js @@ -145,7 +145,7 @@ function standardiseGeographicInfo(geographicInfo) { return geographicInfo; } -db.site.find({"site.extent.geometry": {$exists: true}}).forEach(site => { +db.site.find({"extent.geometry": {$exists: true}}).forEach(site => { let updated = false; let geometry = site.extent && site.extent.geometry; if (geometry) { From 4c32c0bcd200c394ffcac2d00cfd196966326b64 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 5 Nov 2024 17:11:52 +1100 Subject: [PATCH 25/46] AtlasOfLivingAustralia/fieldcapture#1724 & #1034 - title case for spatial names --- .../au/org/ala/ecodata/SpatialService.groovy | 20 +++++++++++++++++-- .../org/ala/ecodata/SpatialServiceSpec.groovy | 8 ++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy index ab30d9074..cf51f62fb 100644 --- a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy @@ -4,10 +4,12 @@ import grails.core.GrailsApplication import grails.plugin.cache.Cacheable import groovy.json.JsonParserType import groovy.json.JsonSlurper -import org.apache.commons.lang.WordUtils import org.locationtech.jts.geom.* import org.locationtech.jts.io.WKTReader +import java.util.regex.Matcher +import java.util.regex.Pattern + import static ParatooService.deepCopy /** * The SpatialService is responsible for: @@ -449,8 +451,22 @@ class SpatialService { synonymLookupTable[facetName] = synonymTable?.collectEntries { k, List v -> v.collectEntries { v1 -> [(v1.toLowerCase()): k] } } } - synonymLookupTable[facetName]?.get(name) ?: WordUtils.capitalize(name) + synonymLookupTable[facetName]?.get(name) ?: titleCase(name) + } + } + + String titleCase(String name) { + Pattern pattern = Pattern.compile("\\b\\w") + Matcher matcher = pattern.matcher(name.toLowerCase()) + StringBuffer capitalizedName = new StringBuffer() + + // Capitalize each matched letter and append it to the result + while (matcher.find()) { + matcher.appendReplacement(capitalizedName, matcher.group().toUpperCase()) } + matcher.appendTail(capitalizedName) + + return capitalizedName.toString() } /** diff --git a/src/test/groovy/au/org/ala/ecodata/SpatialServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SpatialServiceSpec.groovy index 76a721cd3..aa23fbf3b 100644 --- a/src/test/groovy/au/org/ala/ecodata/SpatialServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SpatialServiceSpec.groovy @@ -95,6 +95,14 @@ class SpatialServiceSpec extends Specification implements ServiceUnitTest> getShape2() 1 * webService.get("/ws/shapes/wkt/456") >> getBoundaryShape() } + + def "titleCase should capitalize the first letter of each word"() { + expect: + service.titleCase("new south wales") == "New South Wales" + service.titleCase("australian capital territory") == "Australian Capital Territory" + service.titleCase("act") == "Act" + service.titleCase("nSw") == "Nsw" + } private Geometry getBoundaryShape() { return GeometryUtils.geoJsonMapToGeometry(mapper.readValue('{' + From 780dd9f7131af93041e95693e354476547ddaff0 Mon Sep 17 00:00:00 2001 From: chrisala Date: Thu, 7 Nov 2024 09:49:51 +1100 Subject: [PATCH 26/46] Added org details / orgMintedIdentifier to downlaods #994 #1039 --- .../reporting/ProjectXlsExporter.groovy | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index 4e323c885..61dcbf853 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -50,6 +50,9 @@ class ProjectXlsExporter extends ProjectExporter { List associatedOrgProjectProperties = (1..3).collect{['associatedOrg_name'+it, 'associatedOrg_organisationId'+it, 'associatedOrg_fromDate'+it, 'associatedOrg_toDate'+it, 'associatedOrg_description'+it]}.flatten() + List organisationDetailsHeaders = ['Project ID', 'Grant ID', 'External ID', 'Program', 'Sub-program', 'Management Unit', 'Project Name', 'Contract name', 'Organisation ID', 'Organisation relationship from date', 'Organisation relationship to date', 'Organisation relationship', 'ABN', 'Organisation name'] + List organisationDetailsProperties = ['projectId', 'project_grantId', 'project_externalId', 'project_associatedProgram', 'project_associatedSubProgram', 'project_managementUnitName', 'project_name', 'name', 'organisationId', 'fromDate', 'toDate', 'description', 'abn', 'organisationName'] + List projectHeaders = projectHeadersWithTerminationReason + associatedOrgProjectHeaders + projectStateHeaders List projectProperties = projectPropertiesWithTerminationReason + associatedOrgProjectProperties + projectStateProperties @@ -127,8 +130,8 @@ class ProjectXlsExporter extends ProjectExporter { List rdpProjectDetailsHeaders=commonProjectHeaders + ["Does this project directly support a priority place?","Supported priority places", "Are First Nations people (Indigenous) involved in the project?", "What is the nature of the involvement?","Project delivery assumptions","Project review, evaluation and improvement methodology"] List rdpProjectDetailsProperties =commonProjectProperties + ["supportsPriorityPlace", "supportedPriorityPlaces", "indigenousInvolved", "indigenousInvolvementType", "projectMethodology", "projectREI"] - List datasetHeader = commonProjectHeaders + ["Dataset Title", "What program outcome does this dataset relate to?", "What primary or secondary investment priorities or assets does this dataset relate to?","Other Investment Priority","Which project service and outcome/s does this data set support?","Is this data being collected for reporting against short or medium term outcome statements?", "Is this (a) a baseline dataset associated with a project outcome i.e. against which, change will be measured, (b) a project progress dataset that is tracking change against an established project baseline dataset or (c) a standalone, foundational dataset to inform future management interventions?","Other Dataset Type","Which project baseline does this data set relate to or describe?","What EMSA protocol was used when collecting the data?", "What types of measurements or observations does the dataset include?","Other Measurement Type","Identify the method(s) used to collect the data", "Describe the method used to collect the data in detail", "Identify any apps used during data collection", "Provide a coordinate centroid for the area surveyed", "First collection date", "Last collection date", "Is this data an addition to existing time-series data collected as part of a previous project, or is being collected as part of a broader/national dataset?", "Has your data been included in the Threatened Species Index?","Date of upload", "Who developed/collated the dataset?", "Has a quality assurance check been undertaken on the data?", "Has the data contributed to a publication?", "Where is the data held?", "For all public datasets, please provide the published location. If stored internally by your organisation, write ‘stored internally'", "What format is the dataset?","What is the size of the dataset (KB)?","Unknown size", "Are there any sensitivities in the dataset?", "Primary source of data (organisation or individual that owns or maintains the dataset)", "Dataset custodian (name of contact to obtain access to dataset)", "Progress", "Is Data Collection Ongoing"] - List datasetProperties = commonProjectProperties + ["name", "programOutcome", "investmentPriorities","otherInvestmentPriority","projectOutcomes", "term", "type", "otherDataSetType","baselines", "protocol", "measurementTypes","otherMeasurementType", "methods", "methodDescription", "collectionApp", "location", "startDate", "endDate", "addition", "threatenedSpeciesIndex","threatenedSpeciesIndexUploadDate", "collectorType", "qa", "published", "storageType", "publicationUrl", "format","sizeInKB","sizeUnknown", "sensitivities", "owner", "custodian", "progress", "dataCollectionOngoing"] + List datasetHeader = commonProjectHeaders + ["Dataset Title", "What program outcome does this dataset relate to?", "What primary or secondary investment priorities or assets does this dataset relate to?","Other Investment Priority","Which project service and outcome/s does this data set support?","Is this data being collected for reporting against short or medium term outcome statements?", "Is this (a) a baseline dataset associated with a project outcome i.e. against which, change will be measured, (b) a project progress dataset that is tracking change against an established project baseline dataset or (c) a standalone, foundational dataset to inform future management interventions?","Other Dataset Type","Which project baseline does this data set relate to or describe?","What EMSA protocol was used when collecting the data?", "What types of measurements or observations does the dataset include?","Other Measurement Type","Identify the method(s) used to collect the data", "Describe the method used to collect the data in detail", "Identify any apps used during data collection", "Provide a coordinate centroid for the area surveyed", "First collection date", "Last collection date", "Is this data an addition to existing time-series data collected as part of a previous project, or is being collected as part of a broader/national dataset?", "Has your data been included in the Threatened Species Index?","Date of upload", "Who developed/collated the dataset?", "Has a quality assurance check been undertaken on the data?", "Has the data contributed to a publication?", "Where is the data held?", "For all public datasets, please provide the published location. If stored internally by your organisation, write ‘stored internally'", "What format is the dataset?","What is the size of the dataset (KB)?","Unknown size", "Are there any sensitivities in the dataset?", "Primary source of data (organisation or individual that owns or maintains the dataset)", "Dataset custodian (name of contact to obtain access to dataset)", "Progress", "Is Data Collection Ongoing", "Technical data from Monitor"] + List datasetProperties = commonProjectProperties + ["name", "programOutcome", "investmentPriorities","otherInvestmentPriority","projectOutcomes", "term", "type", "otherDataSetType","baselines", "protocol", "measurementTypes","otherMeasurementType", "methods", "methodDescription", "collectionApp", "location", "startDate", "endDate", "addition", "threatenedSpeciesIndex","threatenedSpeciesIndexUploadDate", "collectorType", "qa", "published", "storageType", "publicationUrl", "format","sizeInKB","sizeUnknown", "sensitivities", "owner", "custodian", "progress", "dataCollectionOngoing", "orgMintedIdentifier"] List electorateInternalOrderNoHeader = (2..3).collect{'Internal order number '+it} List electorateInternalOrderNoProperties = (1..2).collect{PROJECT_DATA_PREFIX+'internalOrderId'+it} @@ -162,6 +165,7 @@ class ProjectXlsExporter extends ProjectExporter { OutputModelProcessor processor = new OutputModelProcessor() ProjectService projectService + OrganisationService organisationService /** Enables us to pre-create headers for each electorate that will appear in the result set */ List distinctElectorates @@ -181,6 +185,7 @@ class ProjectXlsExporter extends ProjectExporter { ProjectXlsExporter(ProjectService projectService, XlsExporter exporter, ManagementUnitService managementUnitService, OrganisationService organisationService, ProgramService programService) { super(exporter) this.projectService = projectService + this.organisationService = organisationService distinctElectorates = new ArrayList() useSpeciesUrlGetter = true setupManagementUnits(managementUnitService) @@ -191,6 +196,7 @@ class ProjectXlsExporter extends ProjectExporter { ProjectXlsExporter(ProjectService projectService, XlsExporter exporter, List tabsToExport, List electorates, ManagementUnitService managementUnitService, OrganisationService organisationService, ProgramService programService, Map downloadMetadata, boolean formSectionPerTab = false) { super(exporter, tabsToExport, [:], TimeZone.default) this.projectService = projectService + this.organisationService = organisationService this.formSectionPerTab = formSectionPerTab useSpeciesUrlGetter = true addDataDescriptionToDownload(downloadMetadata) @@ -285,6 +291,7 @@ class ProjectXlsExporter extends ProjectExporter { addProjectGeo(project) exportProject(project) + exportProjectOrganisationData(project) exportOutputTargets(project) exportSites(project) exportDocuments(project) @@ -356,6 +363,8 @@ class ProjectXlsExporter extends ProjectExporter { } } + + void exportActivities(Map project) { tabsToExport.each { tab -> List activities = project?.activities?.findAll { it.type == tab } @@ -410,6 +419,28 @@ class ProjectXlsExporter extends ProjectExporter { } } + void exportProjectOrganisationData(Map project) { + String sheetName = 'Organisation Details' + if (shouldExport(sheetName)) { + AdditionalSheet sitesSheet = getSheet(sheetName, organisationDetailsProperties, organisationDetailsHeaders) + List associatedOrgs = [] + + project.associatedOrgs?.each { org -> + Map orgProps = org+project + if (org.organisationId) { + Map organisation = organisationService.get(org.organisationId) + orgProps['abn'] = organisation?.abn + orgProps['organisationName'] = organisation?.name + } + + associatedOrgs << orgProps + + } + int row = sitesSheet.getSheet().lastRowNum + sitesSheet.add(associatedOrgs, organisationDetailsProperties, row + 1) + } + } + private void exportSites(Map project) { String sheetName = 'Sites' if (shouldExport(sheetName)) { From 542071e270818cf12903e6d7d602813ecbda3415 Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 18 Nov 2024 07:43:01 +1100 Subject: [PATCH 27/46] Added project dates / renamed org column in downloads #994 --- .../ala/ecodata/reporting/ProjectXlsExporter.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index 61dcbf853..f259a375f 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -28,10 +28,10 @@ class ProjectXlsExporter extends ProjectExporter { List configurableIntersectionHeaders = getIntersectionHeaders() List configurableIntersectionProperties = getIntersectionProperties() - List commonProjectHeadersWithoutSites = ['Project ID', 'Grant ID', 'External ID', 'Internal order number', 'Work order id', 'Recipient (Contract name)', 'Recipient (ID)', 'Management Unit', 'Name', 'Description', 'Program', 'Sub-program', 'Start Date', 'End Date', 'Contracted Start Date', 'Contracted End Date', 'Funding', 'Funding Type', 'Status', "Last Modified"] + configurableIntersectionHeaders + List commonProjectHeadersWithoutSites = ['Project ID', 'Grant ID', 'External ID', 'Internal order number', 'Work order id', 'Contracted recipient name', 'Recipient (ID)', 'Management Unit', 'Name', 'Description', 'Program', 'Sub-program', 'Start Date', 'End Date', 'Contracted Start Date', 'Contracted End Date', 'Funding', 'Funding Type', 'Status', "Last Modified"] + configurableIntersectionHeaders List commonProjectPropertiesRaw = ['grantId', 'externalId', 'internalOrderId', 'workOrderId', 'organisationName', 'organisationId', 'managementUnitName', 'name', 'description', 'associatedProgram', 'associatedSubProgram', 'plannedStartDate', 'plannedEndDate', 'contractStartDate', 'contractEndDate', 'funding', 'fundingType', 'status', 'lastUpdated'] + configurableIntersectionProperties - List projectHeadersWithTerminationReason = ['Project ID', 'Grant ID', 'External ID', 'Internal order number', 'Work order id', 'Recipient (Contract name)', 'Recipient (ID)', 'Management Unit', 'Name', 'Description', 'Program', 'Sub-program', 'Start Date', 'End Date', 'Contracted Start Date', 'Contracted End Date', 'Funding', 'Funding Type', 'Status'] + configurableIntersectionHeaders + ['Termination Reason', 'Last Modified'] + List projectHeadersWithTerminationReason = ['Project ID', 'Grant ID', 'External ID', 'Internal order number', 'Work order id', 'Contracted recipient name', 'Recipient (ID)', 'Management Unit', 'Name', 'Description', 'Program', 'Sub-program', 'Start Date', 'End Date', 'Contracted Start Date', 'Contracted End Date', 'Funding', 'Funding Type', 'Status'] + configurableIntersectionHeaders + ['Termination Reason', 'Last Modified'] List projectPropertiesTerminationReason = ['grantId', 'externalId', 'internalOrderId', 'workOrderId', 'organisationName', 'organisationId', 'managementUnitName', 'name', 'description', 'associatedProgram', 'associatedSubProgram', 'plannedStartDate', 'plannedEndDate', 'contractStartDate', 'contractEndDate', 'funding', 'fundingType', 'status'] + configurableIntersectionProperties List projectPropertiesWithTerminationReason = ['projectId'] + projectPropertiesTerminationReason.collect{PROJECT_DATA_PREFIX+it} + ["terminationReason", PROJECT_DATA_PREFIX+"lastUpdated"] @@ -45,13 +45,13 @@ class ProjectXlsExporter extends ProjectExporter { List commonProjectHeaders = commonProjectHeadersWithoutSites + stateHeaders + electorateHeaders + projectApprovalHeaders List commonProjectProperties = commonProjectPropertiesWithoutSites + stateProperties + electorateProperties + projectApprovalProperties - List associatedOrgProjectHeaders = (1..3).collect{['Contract name '+it, 'Organisation ID '+it, 'Organisation relationship from date '+it, 'Organisation relationship to date '+it, 'Organisation relationship '+it]}.flatten() + List associatedOrgProjectHeaders = (1..3).collect{['Contracted recipient name '+it, 'Organisation ID '+it, 'Organisation relationship from date '+it, 'Organisation relationship to date '+it, 'Organisation relationship '+it]}.flatten() List associatedOrgProperties = ['name', 'organisationId', 'fromDate', 'toDate', 'description'] List associatedOrgProjectProperties = (1..3).collect{['associatedOrg_name'+it, 'associatedOrg_organisationId'+it, 'associatedOrg_fromDate'+it, 'associatedOrg_toDate'+it, 'associatedOrg_description'+it]}.flatten() - List organisationDetailsHeaders = ['Project ID', 'Grant ID', 'External ID', 'Program', 'Sub-program', 'Management Unit', 'Project Name', 'Contract name', 'Organisation ID', 'Organisation relationship from date', 'Organisation relationship to date', 'Organisation relationship', 'ABN', 'Organisation name'] - List organisationDetailsProperties = ['projectId', 'project_grantId', 'project_externalId', 'project_associatedProgram', 'project_associatedSubProgram', 'project_managementUnitName', 'project_name', 'name', 'organisationId', 'fromDate', 'toDate', 'description', 'abn', 'organisationName'] + List organisationDetailsHeaders = ['Project ID', 'Grant ID', 'External ID', 'Program', 'Sub-program', 'Management Unit', 'Project Name', 'Project start date', 'Project end date', 'Contracted recipient name', 'Organisation ID', 'Organisation relationship from date', 'Organisation relationship to date', 'Organisation relationship', 'ABN', 'MERIT organisation name'] + List organisationDetailsProperties = ['projectId', 'project_grantId', 'project_externalId', 'project_associatedProgram', 'project_associatedSubProgram', 'project_managementUnitName', 'project_name', 'project_plannedStartDate', 'project_plannedEndDate', 'name', 'organisationId', 'fromDate', 'toDate', 'description', 'abn', 'organisationName'] List projectHeaders = projectHeadersWithTerminationReason + associatedOrgProjectHeaders + projectStateHeaders List projectProperties = projectPropertiesWithTerminationReason + associatedOrgProjectProperties + projectStateProperties From 4106997fc5222f9d09ed7661162d2f44df01768e Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 18 Nov 2024 09:07:12 +1100 Subject: [PATCH 28/46] Blank MERIT org name from unlinked orgs in download #994 --- .../au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index f259a375f..6054449ba 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -432,6 +432,9 @@ class ProjectXlsExporter extends ProjectExporter { orgProps['abn'] = organisation?.abn orgProps['organisationName'] = organisation?.name } + else { + orgProps['organisationName'] = '' + } associatedOrgs << orgProps From ec5dcbe78d75829582bbfd176f2a9eed7e2998f3 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 22 Nov 2024 16:40:59 +1100 Subject: [PATCH 29/46] AtlasOfLivingAustralia/fieldcapture#3369 added custom field --- grails-app/domain/au/org/ala/ecodata/Organisation.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy index 6b41a63ee..e886b305c 100644 --- a/grails-app/domain/au/org/ala/ecodata/Organisation.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Organisation.groovy @@ -1,10 +1,8 @@ package au.org.ala.ecodata import au.org.ala.ecodata.graphql.mappers.OrganisationGraphQLMapper -import au.org.ala.ecodata.graphql.mappers.ProjectGraphQLMapper import org.bson.types.ObjectId import org.springframework.validation.Errors - /** * Represents an organisation that manages projects in MERIT and BioCollect. * Allows some branding as well as grouping / ownership of projects. @@ -39,6 +37,8 @@ class Organisation { /** Stores configuration information for how reports should be generated for this organisation (if applicable) */ Map config + /** Stores service target details like MeriPlan in projects */ + Map custom String collectoryInstitutionId // Reference to the Collectory @@ -74,6 +74,7 @@ class Organisation { config nullable: true sourceSystem nullable: true externalIds nullable: true + custom nullable: true hubId nullable: true, validator: { String hubId, Organisation organisation, Errors errors -> GormMongoUtil.validateWriteOnceProperty(organisation, 'organisationId', 'hubId', errors) } From a1758003bfe1ba0bbca0ede09196fc98c8ebbe09 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 16 Jun 2023 23:04:52 +1000 Subject: [PATCH 30/46] AtlasOfLivingAustralia/biocollect#1546 - fixed project activity web service - added data view for a survey --- .../org/ala/ecodata/ProjectActivityController.groovy | 3 ++- .../au/org/ala/ecodata/ElasticSearchService.groovy | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/ProjectActivityController.groovy b/grails-app/controllers/au/org/ala/ecodata/ProjectActivityController.groovy index a81b62b7a..d72577acb 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ProjectActivityController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ProjectActivityController.groovy @@ -52,7 +52,8 @@ class ProjectActivityController { } if (!result) { - result = [status: 404, text: 'Invalid id']; + render status: 404, text: [message: 'Invalid id', status: 404] as JSON + return } } diff --git a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy index 7cc950c7d..58d12a78f 100644 --- a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy @@ -1660,6 +1660,17 @@ class ElasticSearchService { } break + case 'projectactivityrecords': + if (projectActivityId) { + if (userId && (permissionService.isUserAlaAdmin(userId) || permissionService.isUserAdminForProject(userId, projectId) || permissionService.isUserEditorForProject(userId, projectId))) { + forcedQuery = '(docType:activity AND projectActivity.projectActivityId:' + projectActivityId + ')' + } + else { + forcedQuery = '(docType:activity AND projectActivity.projectActivityId:' + projectActivityId + ' AND projectActivity.embargoed:false AND (verificationStatusFacet:approved OR verificationStatusFacet:\"not applicable\" OR (NOT _exists_:verificationStatus)))' + } + } + break + case 'myprojectrecords': if (projectId) { if (userId) { From 2925acef60e018f8eead6588f1309cb25b94ab6a Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Tue, 1 Aug 2023 09:41:13 +1000 Subject: [PATCH 31/46] Implemented project activities indexing for the home page / project search WS endpoint --- .../services/au/org/ala/ecodata/ElasticSearchService.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy index 58d12a78f..363c64322 100644 --- a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy @@ -1129,7 +1129,7 @@ class ElasticSearchService { // MERIT project needs private sites to be indexed for faceting purposes but Biocollect does not require private sites. // Some Biocollect project have huge numbers of private sites. This will significantly hurt performance. // Hence the if condition. - if(projectMap.isMERIT){ + if (projectMap.isMERIT) { // Allow ESP sites to be hidden, even on the project explorer. Needs to be tided up a bit as MERIT sites were // already marked as private to avoid discovery via BioCollect @@ -1166,6 +1166,7 @@ class ElasticSearchService { // todo: Check if BioCollect requires all sites in `sites` property. If no, merge `projectArea` with `sites`. projectMap.projectArea = siteService.getSimpleProjectArea(projectMap.projectSiteId) projectMap.containsActivity = activityService.searchAndListActivityDomainObjects([projectId: projectMap.projectId], null, null, null, [max: 1, offset: 0])?.totalCount > 0 + projectMap.activities = activityService.findAllForProjectId(project.projectId, LevelOfDetail.NO_OUTPUTS.name()).findAll({ it.status == "active" }) } projectMap.sites?.each { site -> // Not useful for the search index and there is a bug right now that can result in invalid POI From 144ddd0a39cd0dad58b802979f1b79ef92da5f49 Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Thu, 3 Aug 2023 11:35:34 +1000 Subject: [PATCH 32/46] Updated homepage project activities indexing to use projectActivityService, project activities now only return a subset of required fields --- .../au/org/ala/ecodata/ElasticSearchService.groovy | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy index 363c64322..a673f11bb 100644 --- a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy @@ -1166,7 +1166,16 @@ class ElasticSearchService { // todo: Check if BioCollect requires all sites in `sites` property. If no, merge `projectArea` with `sites`. projectMap.projectArea = siteService.getSimpleProjectArea(projectMap.projectSiteId) projectMap.containsActivity = activityService.searchAndListActivityDomainObjects([projectId: projectMap.projectId], null, null, null, [max: 1, offset: 0])?.totalCount > 0 - projectMap.activities = activityService.findAllForProjectId(project.projectId, LevelOfDetail.NO_OUTPUTS.name()).findAll({ it.status == "active" }) + projectMap.projectActivities = projectActivityService.getAllByProject(project.projectId).collect({ + [ + id: it.id, + projectId: it.projectId, + projectActivityId: it.projectActivityId, + name: it.name, + startDate: it.startDate, + endDate: it.endDate, + ] + }) } projectMap.sites?.each { site -> // Not useful for the search index and there is a bug right now that can result in invalid POI From 1e3e3e6669cf8adff489fc90bc62fb1fa4974c58 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 27 Nov 2024 14:01:07 +1100 Subject: [PATCH 33/46] migrated code necessary for PWA from feature/cognito branch --- grails-app/conf/application.groovy | 2 +- grails-app/conf/data/mapping.json | 7 + .../org/ala/ecodata/ApiKeyInterceptor.groovy | 2 +- .../org/ala/ecodata/AuditInterceptor.groovy | 18 +- .../au/org/ala/ecodata/UserService.groovy | 30 ++ .../updateDataResourceIdForProjects.js | 439 ++++++++++++++++++ scripts/misc/bioCollectProjectReport.js | 48 ++ .../ElasticSearchIndexServiceSpec.groovy | 3 +- 8 files changed, 530 insertions(+), 19 deletions(-) create mode 100644 scripts/data_migration/updateDataResourceIdForProjects.js create mode 100644 scripts/misc/bioCollectProjectReport.js diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index c7e1fa5c2..c1b58c3c8 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -654,7 +654,7 @@ environments { ecodata.use.uuids = false app.external.model.dir = "./models/" grails.serverURL = "http://localhost:8080" - app.uploads.url = "${grails.serverURL}/document/download?filename=" + app.uploads.url = "/document/download/" app.elasticsearch.indexOnGormEvents = true app.elasticsearch.indexAllOnStartup = true diff --git a/grails-app/conf/data/mapping.json b/grails-app/conf/data/mapping.json index 39e8c094a..93c7b44ae 100644 --- a/grails-app/conf/data/mapping.json +++ b/grails-app/conf/data/mapping.json @@ -96,6 +96,13 @@ "type" : "text", "copy_to": ["organisationName","organisationFacet", "organisationSort"] }, + "lastUpdated": { + "type" : "date", + "copy_to" : "lastUpdatedSort" + }, + "lastUpdatedSort" : { + "type" : "keyword", "normalizer" : "case_insensitive_sort" + }, "associatedOrgs": { "properties" : { "name" : { diff --git a/grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy b/grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy index 85814cc40..4226a2db0 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy @@ -41,7 +41,7 @@ class ApiKeyInterceptor { PreAuthorise pa = method.getAnnotation(PreAuthorise) ?: controllerClass.getAnnotation(PreAuthorise) if (pa.basicAuth()) { - au.org.ala.web.UserDetails user = userService.getUserFromJWT() + def user = userService.setUser() request.userId = user?.userId if(permissionService.isUserAlaAdmin(request.userId)) { /* Don't enforce check for ALA admin.*/ diff --git a/grails-app/controllers/au/org/ala/ecodata/AuditInterceptor.groovy b/grails-app/controllers/au/org/ala/ecodata/AuditInterceptor.groovy index 3ae4e7b00..d65ad317c 100644 --- a/grails-app/controllers/au/org/ala/ecodata/AuditInterceptor.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/AuditInterceptor.groovy @@ -7,7 +7,7 @@ import grails.config.Config class AuditInterceptor implements GrailsConfigurationAware { int order = 100 // This needs to be after the @RequireApiKey interceptor which makes the userId available via the authService - String httpRequestHeaderForUserId + static String httpRequestHeaderForUserId UserService userService AuthService authService @@ -16,21 +16,7 @@ class AuditInterceptor implements GrailsConfigurationAware { } boolean before() { - // userId is set from either the request param userId or failing that it tries to get it from - // the UserPrincipal (assumes ecodata is being accessed directly via admin page) - def userId = authService.getUserId() ?: request.getHeader(httpRequestHeaderForUserId) - if (userId) { - def userDetails = userService.setCurrentUser(userId) - if (userDetails) { - // We set the current user details in the request scope because - // the 'afterView' hook can be called prior to the actual rendering (despite the name) - // and the thread local can get clobbered before it is actually required. - // Consumers who have access to the request can simply extract current user details - // from there rather than use the service. - request.setAttribute(UserDetails.REQUEST_USER_DETAILS_KEY, userDetails) - } - } - + userService.setUser() true } diff --git a/grails-app/services/au/org/ala/ecodata/UserService.groovy b/grails-app/services/au/org/ala/ecodata/UserService.groovy index bb841f529..7cbe5e405 100644 --- a/grails-app/services/au/org/ala/ecodata/UserService.groovy +++ b/grails-app/services/au/org/ala/ecodata/UserService.groovy @@ -213,4 +213,34 @@ class UserService { return null } } + + def setUser() { + String userId + GrailsWebRequest grailsWebRequest = GrailsWebRequest.lookup() + HttpServletRequest request = grailsWebRequest.getCurrentRequest() + def userDetails = request.getAttribute(UserDetails.REQUEST_USER_DETAILS_KEY) + + if (userDetails) + return userDetails + + // userId is set from either the request param userId or failing that it tries to get it from + // the UserPrincipal (assumes ecodata is being accessed directly via admin page) + userId = getUserFromJWT() ?: authService.getUserId() ?: request.getHeader(AuditInterceptor.httpRequestHeaderForUserId) + if (!userId) { + userId = request.getHeader(AuditInterceptor.httpRequestHeaderForUserId) + } + + if (userId) { + userDetails = setCurrentUser(userId) + if (userDetails) { + // We set the current user details in the request scope because + // the 'afterView' hook can be called prior to the actual rendering (despite the name) + // and the thread local can get clobbered before it is actually required. + // Consumers who have access to the request can simply extract current user details + // from there rather than use the service. + request.setAttribute(UserDetails.REQUEST_USER_DETAILS_KEY, userDetails) + return userDetails + } + } + } } diff --git a/scripts/data_migration/updateDataResourceIdForProjects.js b/scripts/data_migration/updateDataResourceIdForProjects.js new file mode 100644 index 000000000..23041241f --- /dev/null +++ b/scripts/data_migration/updateDataResourceIdForProjects.js @@ -0,0 +1,439 @@ +var dataProviderId = 'dp3903', + projects = [ + { + "name": "PAW Nestbox Monitoring", + "projectId": "709a7e77-105f-4ba9-9d79-7b087afa43a0", + "collectoryId": "dr19104" + }, + { + "name": "Blackberry biocontrol", + "projectId": "ef09ff0d-2af2-4c58-bee7-3198d4f6db4f", + "collectoryId": "dr18250" + }, + { + "name": "Bringing back the Richmond Birdwing Butterfly to Brisbane", + "projectId": "8aed164a-e3e2-4615-985a-dff45dcc968c", + "collectoryId": "dr19731" + }, + { + "name": "Gorse biocontrol", + "projectId": "f8b5dfbf-1eca-4091-be68-0a958292ba66", + "collectoryId": "dr18278" + }, + { + "name": "Spear thistle biocontrol", + "projectId": "c5b3564b-b170-4105-9e37-6c3de8219e7b", + "collectoryId": "dr18249" + }, + { + "name": "English (scotch) broom biocontrol", + "projectId": "cfd68ff5-10d1-4e52-ae39-4161c9b74c8f", + "collectoryId": "dr18277" + }, + { + "name": "Bridal creeper biocontrol", + "projectId": "450e6de7-95b2-4d24-b3ce-b4e2d3d7e029", + "collectoryId": "dr18276" + }, + { + "name": "Docks biocontrol", + "projectId": "4dd17bc6-6ca8-4551-90d3-362b0210c682", + "collectoryId": "dr18256" + }, + { + "name": "Redlands Weed Spotters", + "projectId": "975552f0-5b47-43d9-8db0-e47dc2199d19", + "collectoryId": "dr19103" + }, + { + "name": "Boneseed biocontrol", + "projectId": "a90d08cf-974d-4441-97e6-af5da34cb118", + "collectoryId": "dr18293" + }, + { + "name": "Onopordum thistle biocontrol", + "projectId": "7b1505b4-dbdf-4ad0-a6db-9068585c02a5", + "collectoryId": "dr18254" + }, + { + "name": "Slender thistle biocontrol ", + "projectId": "56491ea9-f10b-45b0-a645-c7674e31be96", + "collectoryId": "dr18275" + }, + { + "name": "Nodding thistle biocontrol", + "projectId": "36f4b025-73b1-4722-893e-0877e022b0dd", + "collectoryId": "dr18274" + }, + { + "name": "Variegated thistle biocontrol", + "projectId": "ba76374c-2859-43ac-ae9a-b6812a05b32c", + "collectoryId": "dr18273" + }, + { + "name": "Cat's claw creeper biocontrol", + "projectId": "68e5faca-a8df-4cea-a310-8db65965a3e9", + "collectoryId": "dr18272" + }, + { + "name": "Madeira vine biocontrol", + "projectId": "6d55e7aa-5992-4105-bf99-842b70e8698e", + "collectoryId": "dr18271" + }, + { + "name": "Horehound biocontrol", + "projectId": "935994ff-957d-423b-9ebb-6eb39930dfae", + "collectoryId": "dr18270" + }, + { + "name": "Cape broom biocontrol", + "projectId": "45361e8c-91ac-4447-b4da-9cd2a8fa0aae", + "collectoryId": "dr18269" + }, + { + "name": "Tutsan biocontrol", + "projectId": "6b786ef0-5503-4784-8d91-b2571b1323ee", + "collectoryId": "dr18268" + }, + { + "name": "Ragwort biocontrol", + "projectId": "38c288ee-01da-425a-9e54-ddaf94f0e748", + "collectoryId": "dr18267" + }, + { + "name": "Blue heliotrope biocontrol", + "projectId": "ba1af974-d733-4bea-882e-9b93a00892d2", + "collectoryId": "dr18266" + }, + { + "name": "Western Ringtail Possum Project", + "projectId": "69a78bc1-d0fd-483c-8fee-b1426c10f1c7", + "collectoryId": "dr19102" + }, + { + "name": "Wheel cactus biocontrol", + "projectId": "26ae9f28-4916-4579-afc0-7a9af3e19e0a", + "collectoryId": "dr18264" + }, + { + "name": "Tiger pear biocontrol", + "projectId": "ffeb44d7-08d0-4627-a31b-6a6365700a18", + "collectoryId": "dr18263" + }, + { + "name": "Opuntia spp. biocontrol", + "projectId": "15185781-1089-4718-9630-b2828e7b855a", + "collectoryId": "dr18262" + }, + { + "name": "Cylindropuntia spp. biocontrol", + "projectId": "738326b6-125b-49a7-b0ed-360c4c3d0062", + "collectoryId": "dr18261" + }, + { + "name": "Harrisia spp. biocontrol", + "projectId": "f298e305-7e48-4fa9-9c73-8e2c759f7d5f", + "collectoryId": "dr18260" + }, + { + "name": "Nature of Eyre Peninsula BioBlitz", + "projectId": "3595e826-4d38-423a-a9f1-683ed6c34e7a", + "collectoryId": "dr19100" + }, + { + "name": "Alan Anderson Walk Trail Kalamunda - Flora and Fauna", + "projectId": "e1e8d799-c4e7-48b8-8f0f-14db1e0c9bff", + "collectoryId": "dr19099" + }, + { + "name": "Australian elm tree project", + "projectId": "240e13b6-4138-40e8-8fe2-c8ff7ffd23e2", + "collectoryId": "dr19098" + }, + { + "name": "Discovery Circle BioBlitz events", + "projectId": "e3edba35-47ff-404b-9c36-43bfb22d4f41", + "collectoryId": "dr19034" + }, + { + "name": "Manky Parrots: Mapping Psittacine Beak and Feather Disease in Australia", + "projectId": "71330a55-a91a-4456-90b7-9de7bd8ac08d", + "collectoryId": "dr19732" + }, + { + "name": "Blue Mountains Eco Monitoring", + "projectId": "998b3582-6eaf-457c-9981-96842f774e19", + "collectoryId": "dr19096" + }, + { + "name": "Rare Flora of the Sydney Basin Bioregion", + "projectId": "c92698e9-9929-4db2-bbf3-48acf6e92508", + "collectoryId": "dr19095" + }, + { + "name": "Leonora District High School biodiversity inventory", + "projectId": "4f83ba09-7f2e-4a47-a9d4-524ce75f2e0a", + "collectoryId": "dr19094" + }, + { + "name": "Greta Valley Nest Box Monitoring ", + "projectId": "b06aee9c-0596-47ca-83d2-9bd5d7f04ccc", + "collectoryId": "dr19093" + }, + { + "name": "Koala Mapping Mackay, Whitsundays & Central Queensland Areas", + "projectId": "760dc5ab-163a-4779-9e09-bca599c9847b", + "collectoryId": "dr20943" + }, + { + "name": "Upside-down Jellyfish in Lake Macquarie", + "projectId": "95235e3b-65b1-4452-bf48-53c2b2157e5f", + "collectoryId": "dr19090" + }, + { + "name": "Bulimba Creek Catchment Nest Box Monitoring project", + "projectId": "efde3b5d-fc96-4c72-88b4-c8ebb1627166", + "collectoryId": "dr19539" + }, + { + "name": "Wildlife Discovery on your Landcare site using infrared wildlife cameras", + "projectId": "337da9cc-ff8c-4f53-a17a-684fd29cecb2", + "collectoryId": "dr19088" + }, + { + "name": "Agnes Water Turtle Monitoring", + "projectId": "e07e486a-7332-41c4-ab94-b32b9eea875b", + "collectoryId": "dr20147" + }, + { + "name": "Nest box monitoring project Walmer South Nature Conservation Reserve", + "projectId": "e3b3e17b-1cf5-43e3-8dfa-50c11955209f", + "collectoryId": "dr19075" + }, + { + "name": "B4C Road Kill Map", + "projectId": "e8e76e00-ce03-4eab-82a1-c90d86c91930", + "collectoryId": "dr19073" + }, + { + "name": "Wildlife Road Tolls", + "projectId": "b311909a-c3ac-4baf-90cf-ded9b235288d", + "collectoryId": "dr19072" + }, + { + "name": "East Ballina Nest Box Project", + "projectId": "cb659b45-6f78-48be-ade4-599eb78b68c2", + "collectoryId": "dr19071" + }, + { + "name": "Australian tree-kangaroo sightings", + "projectId": "c0c7f4c3-2712-4060-a921-d12f877b8cda", + "collectoryId": "dr19070" + }, + { + "name": "City of Parramatta Wildlife Survey", + "projectId": "34a67738-156b-41bd-834d-1262b5eac86a", + "collectoryId": "dr19069" + }, + { + "name": "Quolls in the Mary River Catchment- A Quoll Seekers Network survey", + "projectId": "ec01e69e-f5db-43bc-8f46-e1653f3b6a53", + "collectoryId": "dr19068" + }, + { + "name": "Mosman Wildlife Sightings ", + "projectId": "d625b411-6c9e-4820-b485-4c32d85e549e", + "collectoryId": "dr19066" + }, + { + "name": "Nest Box Monitoring", + "projectId": "02e5b36a-e7bd-4cb9-8b25-6bf08b95755a", + "collectoryId": "dr19067" + }, + { + "name": "Darebin Creek ", + "projectId": "22b185ee-d0e4-4a2d-b2b4-6a5f753317e4", + "collectoryId": "dr19065" + }, + { + "name": "Entangled Wildlife Australia", + "projectId": "23409a0b-6873-40a1-b291-f7eff3cbe837", + "collectoryId": "dr21370" + }, + { + "name": "Outdoor School - 15 Mile Creek ", + "projectId": "4a55b0c1-64a0-4c2a-9be7-63bbce430ca6", + "collectoryId": "dr19062" + }, + { + "name": "African thistle (Berkheya rigida) in Wyndham", + "projectId": "3fada699-7e26-46e8-a970-b748235ac4ad", + "collectoryId": "dr19061" + }, + { + "name": "Artichoke Thistle Monitoring (Cynara cardunculus)", + "projectId": "dcd863e5-1b3e-4ab8-9a7d-6f5081b8b13b", + "collectoryId": "dr19060" + }, + { + "name": "Cape Tulip (Moraea flaccida and Moraea miniata) in Wyndham", + "projectId": "00609186-21c6-4eff-8271-3f21ea3d59d3", + "collectoryId": "dr19059" + }, + { + "name": "Tree Hollows in Wyndham", + "projectId": "a19df0ed-f12d-433b-946c-e4581d3ed658", + "collectoryId": "dr19058" + }, + { + "name": "Brisbane‚Äôs Big Butterfly Count", + "projectId": "aa5b05c5-041e-4e7d-9f6c-60feee565dfb", + "collectoryId": "dr19057" + }, + { + "name": "Koala Sightings Toowoomba Region", + "projectId": "42e66c41-a0b0-45a5-bbfa-2ad0fd07f1ad", + "collectoryId": "dr19056" + }, + { + "name": "Ironbark Gully Nature Revealed", + "projectId": "126acd61-5053-49c4-b611-a3ea540abe01", + "collectoryId": "dr19055" + }, + { + "name": "Little River Victoria - Tiger Pear monitoring program", + "projectId": "3d2e9184-794c-4c90-8fbb-fc0390aca796", + "collectoryId": "dr19054" + }, + { + "name": "St Kilda Mangrove and Saltmarsh monitoring", + "projectId": "c2b16cf9-d4ac-49c7-b70d-53a13645e044", + "collectoryId": "dr19052" + }, + { + "name": "Burnett Koala Program", + "projectId": "322fa927-005f-4309-b569-8d2294f77c72", + "collectoryId": "dr19725" + }, + { + "name": "Indigo Creek nest box project", + "projectId": "c3c93a06-23cf-4e60-bfd9-e9ffcaf24fd8", + "collectoryId": "dr22046" + }, + { + "name": "Getting Curious - Wetland Birds", + "projectId": "ee777788-b430-4a69-922d-c15540e7ea38", + "collectoryId": "dr19048" + }, + { + "name": "Estuarine, salt lake and saltfield records (with occasional freshwater records and other terrestrial habitat flora records)", + "projectId": "665b17e8-c950-4785-b0c3-e7cc89def22b", + "collectoryId": "dr18188" + }, + { + "name": "Community monitoring of pools and watercourses in the Barossa", + "projectId": "f726eb14-bc2e-4eb2-a52b-7cae996c1029", + "collectoryId": "dr19047" + }, + { + "name": "Community monitoring of pools and watercourses in the Clare Valley", + "projectId": "bf63c385-ddf8-4d60-98f5-a906c6eedf4a", + "collectoryId": "dr19046" + }, + { + "name": "Rocky Shore Citizen Science", + "projectId": "1f9b7a1f-279c-4b27-87a6-bee0da748158", + "collectoryId": "dr19045" + }, + { + "name": "NE NSW Grey-headed Flying-fox Habitat Restoration Project", + "projectId": "4dedb5e9-a14c-46ac-98f8-3fbc1d9aff2d", + "collectoryId": "dr20597" + }, + { + "name": "Northern Tablelands Koala Habitat Restoration Project", + "projectId": "ade6b7ab-118c-4a4d-a9a7-6cde01703e92", + "collectoryId": "dr20599" + }, + { + "name": "Sunshine Coast's Big Butterfly Count", + "projectId": "2e519934-0f8a-4e81-a255-d62a3e366516", + "collectoryId": "dr19044" + }, + { + "name": "STREAM- Harrison's Creek Water Quality Monitoring", + "projectId": "c751258a-074f-4658-8b39-e83f78927930", + "collectoryId": "dr19043" + }, + { + "name": "Boonah and District Landcare tube nest project", + "projectId": "334a0530-30d1-411c-a1df-c4e4ea08b6b3", + "collectoryId": "dr20581" + }, + { + "name": "Flora and Fauna of Penrith", + "projectId": "7e356b40-9d48-466c-8549-c08d749aff06", + "collectoryId": "dr19033" + }, + { + "name": "Waterwatch Murraylands and Riverland", + "projectId": "7d2cb9d1-4013-40bd-9327-8853bd71c0b8", + "collectoryId": "dr22084" + }, + { + "name": "Clarence Valley Koala Habitat Restoration Project", + "projectId": "d5c9bd39-ef23-43de-a5a5-5dc31dec6194", + "collectoryId": "dr20598" + }, + { + "name": "Flora Connections", + "projectId": "aa06ceaf-f84a-4f3a-8e45-6ea5e8a7081d", + "collectoryId": "dr19940" + }, + { + "name": "Superb City Wrens", + "projectId": "51a2762f-7cd9-4330-94be-a1a71c205766", + "collectoryId": "dr20483" + }, + { + "name": "Willoughby Wildlife Watch", + "projectId": "a79ef701-85b4-43d1-a778-d77aa661340f", + "collectoryId": "dr19039" + }, + { + "name": "Seed collection for North Coast Local Land Services", + "projectId": "66a8710f-92e0-4770-8ae3-7e832317049e", + "collectoryId": "dr19435" + }, + { + "name": "BioCondition Lite Audits 202122 Ausecology", + "projectId": "996fce1d-a8a1-4f61-ba94-93d3b3a2984c", + "collectoryId": "dr19036" + }, + { + "name": "Jungarra Ngarrian Conservation Project", + "projectId": "723c1751-0449-4a48-a023-c127197a6d3d", + "collectoryId": "dr19136" + }, + { + "name": "NSW Flying-Fox Habitat Restoration Program", + "projectId": "42a524b8-6164-42b5-9c41-73cc9041c777", + "collectoryId": "dr20600" + }, + { + "name": "Northern Rivers 2022 Flood Assessments", + "projectId": "74da0f68-0b50-46f7-9078-f2ce3d22eebf", + "collectoryId": "dr22012" + } + ], + counter = 0; +print("Updating " + projects.length + " projects"); +projects.forEach(function (project) { + var result = db.project.updateOne({projectId: project.projectId}, {$set: {dataProviderId: dataProviderId, dataResourceId: project.collectoryId}}); + if (result.modifiedCount == 0) + print("Project not modified: " + project.name + " (" + project.projectId + ")"); + else + counter += result.modifiedCount; +}); + +print(counter + ' projects updated'); \ No newline at end of file diff --git a/scripts/misc/bioCollectProjectReport.js b/scripts/misc/bioCollectProjectReport.js new file mode 100644 index 000000000..79cb201dd --- /dev/null +++ b/scripts/misc/bioCollectProjectReport.js @@ -0,0 +1,48 @@ +var projects = db.project.find({isMERIT: false, status: {$ne: 'deleted'}}), + project; +var headers = ["Project start date", "Project end date", "Project type", "Project name", "Project status", + "Count of number of members/participants (including casual project participants) for each project", + "Count of number of surveys for each project", + "Count of number of event records/activities for each project", + "Count of the occurrence records (total) for each project"] + +print(headers.join(",")); +while (projects.hasNext()) { + project = projects.next(); + var columns = [] + + columns.push('"' + getDate(project.plannedStartDate) + '"'); + columns.push('"' + getDate(project.plannedEndDate) + '"'); + columns.push('"' + project.projectType + '"'); + columns.push('"' + project.name.replace('"', '""') + '"' ); + columns.push('"' + project.status + '"'); + columns.push(getMembersOfProjectCount(project)); + columns.push(getSurveysCount(project)); + columns.push(getActivitiesCount(project) ); + columns.push(getOccurrencesCount(project)); + print(columns.join(",")); +} + +function getMembersOfProjectCount(project) { + return db.userPermission.find({entityId: project.projectId, entityType: "au.org.ala.ecodata.Project", status: {$ne: 'deleted'}}).count(); +} + +function getSurveysCount(project) { + return db.projectActivity.find({projectId: project.projectId, status: {$ne: 'deleted'}}).count(); +} + +function getActivitiesCount(project) { + return db.activity.find({projectId: project.projectId, status: {$ne: 'deleted'}}).count(); +} + +function getOccurrencesCount(project) { + return db.record.find({projectId: project.projectId, status: {$ne: 'deleted'}}).count(); +} + +function getDate(date){ + if (date) { + var eventDate = date; + return eventDate.getDate() + '/' + (eventDate.getMonth() + 1) + '/' + eventDate.getFullYear(); + } + else return ""; +} \ No newline at end of file diff --git a/src/test/groovy/au/org/ala/ecodata/ElasticSearchIndexServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ElasticSearchIndexServiceSpec.groovy index 3d03f6b74..8eed80562 100644 --- a/src/test/groovy/au/org/ala/ecodata/ElasticSearchIndexServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ElasticSearchIndexServiceSpec.groovy @@ -5,7 +5,6 @@ import grails.test.mongodb.MongoSpec import grails.testing.services.ServiceUnitTest import grails.web.servlet.mvc.GrailsParameterMap import groovy.json.JsonSlurper -import org.elasticsearch.action.get.GetRequest import org.elasticsearch.action.get.GetResponse import org.elasticsearch.action.index.IndexRequest import org.elasticsearch.action.index.IndexRequestBuilder @@ -23,6 +22,7 @@ class ElasticSearchIndexServiceSpec extends MongoSpec implements ServiceUnitTest PermissionService permissionService = Stub(PermissionService) ProgramService programService = Stub(ProgramService) ProjectService projectService = Mock(ProjectService) + ProjectActivityService projectActivityService = Mock(ProjectActivityService) RestHighLevelClient client = GroovyMock(RestHighLevelClient) // Need a groovy mock here due to final methods SiteService siteService = Mock(SiteService) ActivityService activityService = Mock(ActivityService) @@ -37,6 +37,7 @@ class ElasticSearchIndexServiceSpec extends MongoSpec implements ServiceUnitTest service.projectService = projectService service.siteService = siteService service.activityService = activityService + service.projectActivityService = projectActivityService service.documentService = documentService JSON.registerObjectMarshaller(new MapMarshaller()) From 8d2f0dd0a8b49fc938834cfd9d6cdeffb019ec4e Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 27 Nov 2024 14:20:24 +1100 Subject: [PATCH 34/46] 5.1-PWA-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 51515114d..7c2cdc65e 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.0.1" +version "5.1-PWA-SNAPSHOT" group "au.org.ala" description "Ecodata" From 5e61defed167a5a0e82e2e409b272e51b6adecad Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 28 Nov 2024 14:13:30 +1100 Subject: [PATCH 35/46] change from code review --- grails-app/services/au/org/ala/ecodata/UserService.groovy | 3 --- 1 file changed, 3 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/UserService.groovy b/grails-app/services/au/org/ala/ecodata/UserService.groovy index 7cbe5e405..515d1f806 100644 --- a/grails-app/services/au/org/ala/ecodata/UserService.groovy +++ b/grails-app/services/au/org/ala/ecodata/UserService.groovy @@ -226,9 +226,6 @@ class UserService { // userId is set from either the request param userId or failing that it tries to get it from // the UserPrincipal (assumes ecodata is being accessed directly via admin page) userId = getUserFromJWT() ?: authService.getUserId() ?: request.getHeader(AuditInterceptor.httpRequestHeaderForUserId) - if (!userId) { - userId = request.getHeader(AuditInterceptor.httpRequestHeaderForUserId) - } if (userId) { userDetails = setCurrentUser(userId) From e957a953d39715d834202a1168dc918a236d93d4 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 3 Dec 2024 08:33:50 +1100 Subject: [PATCH 36/46] Update build.gradle --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2c18d5340..a5c471884 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.1-SNAPSHOT" +version "5.1-test-SNAPSHOT" group "au.org.ala" description "Ecodata" From b50ebab7c4683b1f97be08dac4fcbca307ffaf60 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 3 Dec 2024 15:00:18 +1100 Subject: [PATCH 37/46] fix for issue with not finding monitor user --- grails-app/services/au/org/ala/ecodata/UserService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/UserService.groovy b/grails-app/services/au/org/ala/ecodata/UserService.groovy index 515d1f806..82c218f51 100644 --- a/grails-app/services/au/org/ala/ecodata/UserService.groovy +++ b/grails-app/services/au/org/ala/ecodata/UserService.groovy @@ -225,7 +225,7 @@ class UserService { // userId is set from either the request param userId or failing that it tries to get it from // the UserPrincipal (assumes ecodata is being accessed directly via admin page) - userId = getUserFromJWT() ?: authService.getUserId() ?: request.getHeader(AuditInterceptor.httpRequestHeaderForUserId) + userId = getUserFromJWT()?.userId ?: authService.getUserId() ?: request.getHeader(AuditInterceptor.httpRequestHeaderForUserId) if (userId) { userDetails = setCurrentUser(userId) From de75e7e55f53373796a2b2f32b5db92c66d34b46 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 4 Dec 2024 08:29:29 +1100 Subject: [PATCH 38/46] Stub spatial due to errors AtlasOfLivingAustralia/fieldcapture#3368 --- grails-app/conf/application.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 46115fef2..f920c50ac 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -700,6 +700,7 @@ environments { audit.thread.schedule.interval = 500l; paratoo.core.baseUrl = "http://localhost:${wiremock.port}/monitor" + spatial.baseUrl = "http://localhost:${wiremock.port}" } production { grails.logging.jul.usebridge = false From b0bc704356ddb4a986897894206e73b88203cf03 Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 9 Dec 2024 10:00:10 +1100 Subject: [PATCH 39/46] Added name identifier for the Score entity AtlasOfLivingAustralia/fieldcapture#3368 --- grails-app/domain/au/org/ala/ecodata/Score.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/grails-app/domain/au/org/ala/ecodata/Score.groovy b/grails-app/domain/au/org/ala/ecodata/Score.groovy index 843e99a1f..f02df1dae 100644 --- a/grails-app/domain/au/org/ala/ecodata/Score.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Score.groovy @@ -13,6 +13,9 @@ class Score { /** The label for this score when displayed */ String label + /** Unique human name for this score to be used as a more descriptive alternative to the scoreId in forms/configuration. Must be unique if present */ + String name + /** A more detailed description of the score and how it should be interpreted */ String description @@ -53,6 +56,7 @@ class Score { tags: nullable:true label unique: true scoreId unique: true + name nullable: true, unique: true } static mapping = { @@ -88,7 +92,8 @@ class Score { entity:entity, externalId:externalId, entityTypes:entityTypes, - tags:tags + tags:tags, + name:name ] if (includeConfig) { scoreMap.configuration = configuration From 49abd9ccaad9574579f8658cd6dec1161e7e80e8 Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 9 Dec 2024 10:01:27 +1100 Subject: [PATCH 40/46] Back to 5.1-SNAPSHOT AtlasOfLivingAustralia/fieldcapture#3368 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a5c471884..2c18d5340 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.1-test-SNAPSHOT" +version "5.1-SNAPSHOT" group "au.org.ala" description "Ecodata" From 5fa40805e45314d32125ce9f42914537b4faff32 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 10 Dec 2024 11:31:42 +1100 Subject: [PATCH 41/46] adding published survey flag to projectActivities --- .../services/au/org/ala/ecodata/ElasticSearchService.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy index a673f11bb..2722d0551 100644 --- a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy @@ -1174,6 +1174,7 @@ class ElasticSearchService { name: it.name, startDate: it.startDate, endDate: it.endDate, + published: it.published ] }) } From e112405bd8f31a1d8b88ccd14d96a337afad2910 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 11 Dec 2024 10:14:54 +1100 Subject: [PATCH 42/46] Fixed poi-ooxml dependencies AtlasOfLivingAustralia/fieldcapture#3384 --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 2c18d5340..7e1996d6a 100644 --- a/build.gradle +++ b/build.gradle @@ -105,6 +105,7 @@ dependencies { implementation 'org.apache.poi:poi:5.2.2' implementation 'org.apache.poi:poi-ooxml:5.2.2' + implementation 'org.apache.poi:poi-ooxml-full:5.2.2' implementation 'org.codehaus.groovy:groovy-dateutil:3.0.8' implementation "org.grails.plugins:ala-auth:$alaSecurityLibsVersion" From 3387de225e35cf82871100161b4c5697855ea14d Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 12 Dec 2024 06:22:38 +1100 Subject: [PATCH 43/46] project activity change trigger update of project --- .../au/org/ala/ecodata/ElasticSearchService.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy index 2722d0551..c6cee7f7c 100644 --- a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy @@ -111,7 +111,7 @@ class ElasticSearchService { RestHighLevelClient client ElasticSearchIndexManager indexManager def indexingTempInactive = false // can be set to true for loading of dump files, etc - def ALLOWED_DOC_TYPES = [Project.class.name, Site.class.name, Document.class.name, Activity.class.name, Record.class.name, Organisation.class.name, UserPermission.class.name, Program.class.name, Output.class.name] + def ALLOWED_DOC_TYPES = [Project.class.name, Site.class.name, Document.class.name, Activity.class.name, Record.class.name, Organisation.class.name, UserPermission.class.name, Program.class.name, Output.class.name, ProjectActivity.class.name] def DEFAULT_FACETS = 10 private static Queue _messageQueue = new ConcurrentLinkedQueue() @@ -642,6 +642,14 @@ class ElasticSearchService { indexHomePage(doc, Project.class.name) } break + case ProjectActivity.class.name: + // make sure updates to project activity updates project object. + // helps BioCollect mobile app show correct surveys. + ProjectActivity projectActivity = ProjectActivity.findByProjectActivityId(docId) + if (projectActivity?.projectId) { + indexDocType(projectActivity.projectId, Project.class.name) + } + break } } From b756005f48e705821cbb0d4d4d9e898dd5dcf31f Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 4 Dec 2024 08:29:29 +1100 Subject: [PATCH 44/46] Stub spatial due to errors AtlasOfLivingAustralia/fieldcapture#3368 --- grails-app/conf/application.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 46115fef2..f920c50ac 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -700,6 +700,7 @@ environments { audit.thread.schedule.interval = 500l; paratoo.core.baseUrl = "http://localhost:${wiremock.port}/monitor" + spatial.baseUrl = "http://localhost:${wiremock.port}" } production { grails.logging.jul.usebridge = false From 1fdd361dfa694dcd7407cfc23ea34ead8646fcc9 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 12 Dec 2024 06:22:38 +1100 Subject: [PATCH 45/46] project activity change trigger update of project --- .../au/org/ala/ecodata/ElasticSearchService.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy index 2722d0551..c6cee7f7c 100644 --- a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy @@ -111,7 +111,7 @@ class ElasticSearchService { RestHighLevelClient client ElasticSearchIndexManager indexManager def indexingTempInactive = false // can be set to true for loading of dump files, etc - def ALLOWED_DOC_TYPES = [Project.class.name, Site.class.name, Document.class.name, Activity.class.name, Record.class.name, Organisation.class.name, UserPermission.class.name, Program.class.name, Output.class.name] + def ALLOWED_DOC_TYPES = [Project.class.name, Site.class.name, Document.class.name, Activity.class.name, Record.class.name, Organisation.class.name, UserPermission.class.name, Program.class.name, Output.class.name, ProjectActivity.class.name] def DEFAULT_FACETS = 10 private static Queue _messageQueue = new ConcurrentLinkedQueue() @@ -642,6 +642,14 @@ class ElasticSearchService { indexHomePage(doc, Project.class.name) } break + case ProjectActivity.class.name: + // make sure updates to project activity updates project object. + // helps BioCollect mobile app show correct surveys. + ProjectActivity projectActivity = ProjectActivity.findByProjectActivityId(docId) + if (projectActivity?.projectId) { + indexDocType(projectActivity.projectId, Project.class.name) + } + break } } From 9d542312b4f6541c20ec71c70ac09ebf125cd9b9 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 12 Dec 2024 14:53:20 +1100 Subject: [PATCH 46/46] ala security plugin 6.3.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 8c8d7566e..c05270c1b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,7 +6,7 @@ gormMongoVersion=8.2.0 grailsViewsVersion=2.3.2 assetPipelineVersion=4.3.0 elasticsearchVersion=7.17.21 -alaSecurityLibsVersion=6.3.0-SNAPSHOT +alaSecurityLibsVersion=6.3.0 #22.x+ causes issues with mongo / GORM javax.validation.spi, might need grails 5 geoToolsVersion=21.5 #jtsVersion must match the geotools version