Preparing v5.1

build.gradle

@@ -20,7 +20,7 @@ plugins {
     id "com.gorylenko.gradle-git-properties" version "2.4.1"
 }
 
-version "5.0.1"
+version "5.1-SNAPSHOT"
 group "au.org.ala"
 description "Ecodata"
 
@@ -105,6 +105,7 @@ dependencies {
 
     implementation 'org.apache.poi:poi:5.2.2'
     implementation 'org.apache.poi:poi-ooxml:5.2.2'
+    implementation 'org.apache.poi:poi-ooxml-full:5.2.2'
     implementation 'org.codehaus.groovy:groovy-dateutil:3.0.8'
 
     implementation "org.grails.plugins:ala-auth:$alaSecurityLibsVersion"

gradle.properties

@@ -6,7 +6,7 @@ gormMongoVersion=8.2.0
 grailsViewsVersion=2.3.2
 assetPipelineVersion=4.3.0
 elasticsearchVersion=7.17.21
-alaSecurityLibsVersion=6.3.0-SNAPSHOT
+alaSecurityLibsVersion=6.3.0
 #22.x+ causes issues with mongo / GORM javax.validation.spi, might need grails 5
 geoToolsVersion=21.5
 #jtsVersion must match the geotools version

grails-app/conf/application.groovy

@@ -496,7 +496,24 @@ app {
             }
             checkForBoundaryIntersectionInLayers = [ "cl927", "cl11163" ]
         }
-        displayNames = [elect: "Electorate(s)", state: "State(s)"]
+        displayNames = [
+                elect: [
+                    headerName: "Electorate(s)"
+                ],
+                state: [
+                        headerName: "State(s)",
+                        mappings: [
+                                "Northern Territory": ["Northern Territory (including Coastal Waters)", "NT"],
+                                "Tasmania": ["Tasmania (including Coastal Waters)", "TAS"],
+                                "New South Wales": ["New South Wales (including Coastal Waters)", "NSW"],
+                                "Victoria": ["Victoria (including Coastal Waters)", "VIC"],
+                                "Queensland": ["Queensland (including Coastal Waters)", "QLD"],
+                                "South Australia": ["South Australia (including Coastal Waters)", "SA"],
+                                "Australian Capital Territory": ["ACT"],
+                                "Western Australia": ["Western Australia (including Coastal Waters)", "WA"]
+                        ]
+                ]
+        ]
     }
 }
 /******************************************************************************\
@@ -654,7 +671,7 @@ environments {
         ecodata.use.uuids = false
         app.external.model.dir = "./models/"
         grails.serverURL = "http://localhost:8080"
-        app.uploads.url = "${grails.serverURL}/document/download?filename="
+        app.uploads.url = "/document/download/"
 
         app.elasticsearch.indexOnGormEvents = true
         app.elasticsearch.indexAllOnStartup = true
@@ -683,6 +700,7 @@ environments {
         audit.thread.schedule.interval = 500l;
 
         paratoo.core.baseUrl = "http://localhost:${wiremock.port}/monitor"
+        spatial.baseUrl = "http://localhost:${wiremock.port}"
     }
     production {
         grails.logging.jul.usebridge = false

grails-app/conf/data/mapping.json

@@ -70,10 +70,6 @@
         "organisationId": {
           "type" : "keyword"
         },
-        "orgIdSvcProvider": {
-          "type" : "keyword",
-          "copy_to": ["organisationId"]
-        },
         "organisationName": {
           "type" : "text",
           "copy_to": ["organisationFacet", "organisationSort"]
@@ -96,6 +92,13 @@
           "type" : "text",
           "copy_to": ["organisationName","organisationFacet", "organisationSort"]
         },
+        "lastUpdated": {
+          "type" : "date",
+          "copy_to" : "lastUpdatedSort"
+        },
+        "lastUpdatedSort" : {
+          "type" : "keyword", "normalizer" : "case_insensitive_sort"
+        },
         "associatedOrgs": {
           "properties" : {
             "name" : {

grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy

@@ -41,7 +41,7 @@ class ApiKeyInterceptor {
             PreAuthorise pa = method.getAnnotation(PreAuthorise) ?: controllerClass.getAnnotation(PreAuthorise)
 
             if (pa.basicAuth()) {
-                au.org.ala.web.UserDetails user = userService.getUserFromJWT()
+                def user = userService.setUser()
                 request.userId = user?.userId
                 if(permissionService.isUserAlaAdmin(request.userId)) {
                     /* Don't enforce check for ALA admin.*/

grails-app/controllers/au/org/ala/ecodata/AuditInterceptor.groovy

@@ -7,7 +7,7 @@ import grails.config.Config
 class AuditInterceptor implements GrailsConfigurationAware {
 
     int order = 100 // This needs to be after the @RequireApiKey interceptor which makes the userId available via the authService
-    String httpRequestHeaderForUserId
+    static String httpRequestHeaderForUserId
     UserService userService
     AuthService authService
 
@@ -16,21 +16,7 @@ class AuditInterceptor implements GrailsConfigurationAware {
     }
 
     boolean before() {
-        // userId is set from either the request param userId or failing that it tries to get it from
-        // the UserPrincipal (assumes ecodata is being accessed directly via admin page)
-        def userId = authService.getUserId() ?: request.getHeader(httpRequestHeaderForUserId)
-        if (userId) {
-            def userDetails = userService.setCurrentUser(userId)
-            if (userDetails) {
-                // We set the current user details in the request scope because
-                // the 'afterView' hook can be called prior to the actual rendering (despite the name)
-                // and the thread local can get clobbered before it is actually required.
-                // Consumers who have access to the request can simply extract current user details
-                // from there rather than use the service.
-                request.setAttribute(UserDetails.REQUEST_USER_DETAILS_KEY, userDetails)
-            }
-        }
-
+        userService.setUser()
         true
     }
 

grails-app/controllers/au/org/ala/ecodata/ProjectActivityController.groovy

@@ -52,7 +52,8 @@ class ProjectActivityController {
             }
 
             if (!result) {
-                result = [status: 404, text: 'Invalid id'];
+                render status: 404, text: [message:  'Invalid id', status: 404] as JSON
+                return
             }
         }
 

grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy

@@ -324,6 +324,15 @@ class ProjectController {
         render result as JSON
     }
 
+    def findStateAndElectorateForProject() {
+        if (!params.projectId) {
+            render status:400, text: "projectId is a required parameter"
+        } else {
+            Map project = projectService.get(params.projectId)
+            asJson projectService.findStateAndElectorateForProject(project)
+        }
+    }
+
     def findByName() {
         if (!params.projectName) {
             render status:400, text: "projectName is a required parameter"

grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy

@@ -0,0 +1,134 @@
+package au.org.ala.ecodata
+
+import au.org.ala.ecodata.spatial.SpatialConversionUtils
+import au.org.ala.ecodata.spatial.SpatialUtils
+import org.apache.commons.fileupload.servlet.ServletFileUpload
+import org.apache.commons.io.IOUtils
+import org.apache.commons.lang3.tuple.Pair
+import org.locationtech.jts.geom.Geometry
+import org.springframework.web.multipart.MultipartFile
+
+import javax.servlet.http.HttpServletResponse
+@au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.readScope"])
+class SpatialController {
+    SpatialService spatialService
+    static responseFormats = ['json', 'xml']
+    static allowedMethods = [uploadShapeFile: "POST", getShapeFileFeatureGeoJson: "GET"]
+
+    @au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.writeScope"])
+    def uploadShapeFile() {
+        // Use linked hash map to maintain key ordering
+        Map<Object, Object> retMap = new LinkedHashMap<Object, Object>()
+
+        File tmpZipFile = File.createTempFile("shpUpload", ".zip")
+
+        if (ServletFileUpload.isMultipartContent(request)) {
+            // Parse the request
+            Map<String, MultipartFile> items = request.getFileMap()
+
+            if (items.size() == 1) {
+                MultipartFile fileItem = items.values()[0]
+                IOUtils.copy(fileItem.getInputStream(), new FileOutputStream(tmpZipFile))
+                retMap.putAll(handleZippedShapeFile(tmpZipFile))
+                response.setStatus(HttpServletResponse.SC_OK)
+            } else {
+                response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
+                retMap.put("error", "Multiple files sent in request. A single zipped shape file should be supplied.")
+            }
+        }
+
+        respond retMap
+    }
+
+    @au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.writeScope"])
+    def getShapeFileFeatureGeoJson() {
+        Map retMap
+        String shapeId = params.shapeFileId
+        String featureIndex = params.featureId
+        if (featureIndex != null && shapeId != null) {
+
+            retMap = processShapeFileFeatureRequest(shapeId, featureIndex)
+            if(retMap.geoJson == null) {
+                response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
+            }
+            else {
+                response.setStatus(HttpServletResponse.SC_OK)
+            }
+        }
+        else {
+            response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
+            retMap = ["error": "featureId and shapeFileId must be supplied"]
+        }
+
+        respond retMap
+    }
+
+    def features() {
+        def retVariable
+        if (!params.layerId) {
+            response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
+            retVariable = ["error": "layerId must be supplied"]
+        }
+        else {
+            List<String> intersectWith = params.intersectWith?.split(",") ?: []
+            retVariable = spatialService.features(params.layerId, intersectWith)
+        }
+
+        respond retVariable
+    }
+
+    private Map<String, String> processShapeFileFeatureRequest(String shapeFileId, String featureIndex) {
+        Map<String, Object> retMap = new HashMap<String, Object>()
+
+        try {
+            File shpFileDir = new File(System.getProperty("java.io.tmpdir"), shapeFileId)
+            Geometry geoJson = SpatialUtils.getShapeFileFeaturesAsGeometry(shpFileDir, featureIndex)
+
+            if (geoJson == null) {
+                retMap.put("error", "Invalid geometry")
+                return retMap
+            }
+            else {
+                if (geoJson.getCoordinates().flatten().size() > grailsApplication.config.getProperty("shapefile.simplify.threshhold", Integer,  50_000)) {
+                    geoJson = GeometryUtils.simplify(geoJson, grailsApplication.config.getProperty("shapefile.simplify.tolerance", Double, 0.0001))
+                }
+
+                retMap.put("geoJson", GeometryUtils.geometryToGeoJsonMap(geoJson, grailsApplication.config.getProperty("shapefile.geojson.decimal", Integer, 20)))
+            }
+        } catch (Exception ex) {
+            log.error("Error processsing shapefile feature request", ex)
+            retMap.put("error", ex.getMessage())
+        }
+
+        return retMap
+    }
+
+    private static Map<Object, Object> handleZippedShapeFile(File zippedShp) throws IOException {
+        // Use linked hash map to maintain key ordering
+        Map<Object, Object> retMap = new LinkedHashMap<Object, Object>()
+
+        Pair<String, File> idFilePair = SpatialConversionUtils.extractZippedShapeFile(zippedShp)
+        String uploadedShpId = idFilePair.getLeft()
+        File shpFile = idFilePair.getRight()
+
+        retMap.put("shp_id", uploadedShpId)
+
+        List<List<Pair<String, Object>>> manifestData = SpatialConversionUtils.getShapeFileManifest(shpFile)
+
+        int featureIndex = 0
+        for (List<Pair<String, Object>> featureData : manifestData) {
+            // Use linked hash map to maintain key ordering
+            Map<String, Object> featureDataMap = new LinkedHashMap<String, Object>()
+
+            for (Pair<String, Object> fieldData : featureData) {
+                featureDataMap.put(fieldData.getLeft(), fieldData.getRight())
+            }
+
+            retMap.put(featureIndex, featureDataMap)
+
+            featureIndex++
+        }
+
+        return retMap
+    }
+}

grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy

@@ -42,6 +42,9 @@ class UrlMappings {
 
         "/ws/output/getOutputSpeciesUUID/"(controller: "output"){ action = [GET:"getOutputSpeciesUUID"] }
 
+        "/ws/shapefile" (controller: "spatial"){ action = [POST:"uploadShapeFile"] }
+        "/ws/shapefile/geojson/$shapeFileId/$featureId"(controller: "spatial"){ action = [GET:"getShapeFileFeatureGeoJson"] }
+
         "/ws/activitiesForProject/$id" {
             controller = 'activity'
             action = 'activitiesForProject'
@@ -195,6 +198,7 @@ class UrlMappings {
         "/ws/project/getBiocollectFacets"(controller: "project"){ action = [GET:"getBiocollectFacets"] }
         "/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets")
         "/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords")
+        "/ws/project/findStateAndElectorateForProject"(controller: "project", action: "findStateAndElectorateForProject")
         "/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch")
         "/ws/dataSetSummary/$projectId/$dataSetId?"(controller :'dataSetSummary') {
 

grails-app/domain/au/org/ala/ecodata/AssociatedOrg.groovy

@@ -10,11 +10,29 @@ import groovy.transform.ToString
 @JsonIgnoreProperties(['metaClass', 'errors', 'expandoMetaClass'])
 class AssociatedOrg {
 
+    /** Reference to the Organisation entity if ecodata has a record of the Organisation */
     String organisationId
+
+    /**
+     * The name of the organisation in the context of the relationship.  e.g. it could be a name used
+     * in a contract with a project that is different from the current business name of the organisation
+     */
     String name
     String logo
     String url
 
+    /**
+     * The date the association started.  A null date indicates the relationship started at the same
+     * time as the related entity. e.g. the start of a Project
+     */
+    Date fromDate
+
+    /**
+     * The date the association e ended.  A null date indicates the relationship ended at the same
+     * time as the related entity. e.g. the end of a Project
+     */
+    Date toDate
+
     /** A description of the association - e.g. Service Provider, Grantee, Sponsor */
     String description
 
@@ -25,8 +43,11 @@ class AssociatedOrg {
         organisationId nullable: true
         name nullable: true
         logo nullable: true
+
         url nullable: true
         description nullable: true
+        fromDate nullable: true
+        toDate nullable: true
     }
 
 }

grails-app/domain/au/org/ala/ecodata/ExternalId.groovy

@@ -12,7 +12,7 @@ class ExternalId implements Comparable {
 
     enum IdType {
         INTERNAL_ORDER_NUMBER, TECH_ONE_CODE, WORK_ORDER, GRANT_AWARD, GRANT_OPPORTUNITY, RELATED_PROJECT,
-        MONITOR_PROTOCOL_INTERNAL_ID, MONITOR_PROTOCOL_GUID, TECH_ONE_CONTRACT_NUMBER, MONITOR_PLOT_GUID,
+        MONITOR_PROTOCOL_INTERNAL_ID, MONITOR_PROTOCOL_GUID, TECH_ONE_CONTRACT_NUMBER, TECH_ONE_PARTY_ID, MONITOR_PLOT_GUID,
         MONITOR_PLOT_SELECTION_GUID, MONITOR_MINTED_COLLECTION_ID, UNSPECIFIED }
 
     static constraints = {

grails-app/domain/au/org/ala/ecodata/GeographicInfo.groovy

@@ -20,6 +20,12 @@ class GeographicInfo {
     /** Some projects don't have specific geographic areas and are flagged as being run nationwide */
     boolean nationwide = false
 
+    /** A flag to indicate that the project is running statewide i.e. all electorates in a state */
+    boolean statewide = false
+
+    /** A flag to override calculated values for states and electorates with manually entered values */
+    boolean isDefault = false
+
     /** The primary state in which this project is running, if applicable */
     String primaryState