update weaver 6.1.0 (#489)

## Overview

Apply latest Weaver to apply all updates and fixes since 5.6.1.

## Changes

**Non-breaking changes**
- Weaver: update `weaver` component default version to [6.1.0](https://github.com/crim-ca/weaver/tree/6.1.0).

  ### Relevant changes
  * Add support of *OGC API - Processes - Part 3: Workflows and Chaining* with *Nested Process* ad-hoc workflow.
  * Add support of *OGC API - Processes - Part 3: Workflows and Chaining* with *Remote Collection* (STAC and OGC).
  * Add support of *OGC API - Processes - Part 4: Job Management* endpoints for job "pending" creation and execution.
  * Add support of *OGC API - Processes - Part 4: Job Management* endpoints for job provenance as *W3C PROV* metadata.
  * Multiple alignment and fixes related to latest *OGC API - Processes - Part 1: Core* definitions regarding handling
    of input parameters and headers when submitting jobs to obtain alternate result representations and behavior.
  * Add HTML responses by default via web browsers or as requested by `Accept` headers or `f` query parameter.
  * Add improved CWL schema validation with `Weaver`-specific definitions where applicable
    (see https://github.com/crim-ca/weaver/tree/master/weaver/schemas/cwl).

- Weaver: modifications to `proxy` configurations for `weaver`

  * Add `WEAVER_ALT_PREFIX` optional variable that auto-configures `WEAVER_ALT_PREFIX_PROXY_LOCATION`,
    which allows setting an alternate endpoint to redirect requests to `weaver`.
    It uses `/ogcapi` by default which is a very common expectation from servers supporting OGC standards.
  * Use the `TWITCHER_VERIFY_PATH` approach to accelerate access of `weaver` resources authorization.
  * Modify proxy pass definitions and URL prefixes to resolve correctly with HTML resources.

**Breaking changes**
- n/a

## Related Issue / Discussion

- Related to OGC Testbed-20 initiatives.

## Additional Information

Links to other issues or sources.

- The PR is live here: https://hirondelle.crim.ca/weaver/

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.7.3
+current_version = 2.8.0
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2025-01-17T18:54:34Z
+search = {current_version} 2025-01-17T23:17:16Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2025-01-17T18:54:34Z
+values = 2025-01-17T23:17:16Z
 
 [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

CHANGES.md

@@ -17,6 +17,32 @@
 
 [//]: # (list changes here, using '-' for each new entry, remove this when items are added)
 
+[2.8.0](https://github.com/bird-house/birdhouse-deploy/tree/2.8.0) (2025-01-17)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Weaver: update `weaver` component default version to [6.1.1](https://github.com/crim-ca/weaver/tree/6.1.1).
+
+  ### Relevant changes
+  * Add support of *OGC API - Processes - Part 3: Workflows and Chaining* with *Nested Process* ad-hoc workflow.
+  * Add support of *OGC API - Processes - Part 3: Workflows and Chaining* with *Remote Collection* (STAC and OGC).
+  * Add support of *OGC API - Processes - Part 4: Job Management* endpoints for job "pending" creation and execution.
+  * Add support of *OGC API - Processes - Part 4: Job Management* endpoints for job provenance as *W3C PROV* metadata.
+  * Multiple alignment and fixes related to latest *OGC API - Processes - Part 1: Core* definitions regarding handling
+    of input parameters and headers when submitting jobs to obtain alternate result representations and behavior.
+  * Add HTML responses by default via web browsers or as requested by `Accept` headers or `f` query parameter.
+  * Add improved CWL schema validation with `Weaver`-specific definitions where applicable
+    (see https://github.com/crim-ca/weaver/tree/master/weaver/schemas/cwl).
+
+- Weaver: modifications to `proxy` configurations for `weaver`
+
+  * Add `WEAVER_ALT_PREFIX` optional variable that auto-configures `WEAVER_ALT_PREFIX_PROXY_LOCATION`,
+    which allows setting an alternate endpoint to redirect requests to `weaver`.
+    It uses `/ogcapi` by default which is a very common expectation from servers supporting OGC standards.
+  * Use the `TWITCHER_VERIFY_PATH` approach to accelerate access of `weaver` resources authorization.
+  * Modify proxy pass definitions and URL prefixes to resolve correctly with HTML resources.
+
 [2.7.3](https://github.com/bird-house/birdhouse-deploy/tree/2.7.3) (2025-01-17)
 ------------------------------------------------------------------------------------------------------------------
 

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 2.7.3
+override APP_VERSION := 2.8.0
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -18,13 +18,13 @@ for a full-fledged production platform.
     * - citation
       - | |citation|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.7.3.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.8.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.7.3...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.8.0...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-2.7.3-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-2.8.0-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.7.3
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.8.0
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-2.7.3 2025-01-17T18:54:34Z
+2.8.0 2025-01-17T23:17:16Z

birdhouse/components/README.rst

@@ -448,6 +448,16 @@ Customizing the Component
     Further ``docker-compose-extra.yml`` could be needed to define
     any other ``volumes`` entries where these component would need to be mounted to.
 
+  - Optionally, set ``WEAVER_ALT_PREFIX`` with any desired prefix location to use as alternate alias
+    for the ``/weaver/`` endpoint. The ``/weaver/`` endpoint will remain available.
+    The ``WEAVER_ALT_PREFIX`` alias defines an *additional* equivalent location to access the service.
+    By default ``/ogcapi`` is employed as a common value for this suite of OGC standards.
+
+    Note that custom prefix values, if specified, should start with a leading ``/``, and leave out any trailing ``/``.
+    The prefix can also use multiple levels as desired (e.g.: ``/my/custom/path``).
+
+    If the original ``/weaver/`` endpoint is deemed sufficient, and you would rather omit this additional alias
+    entirely, the ``WEAVER_ALT_PREFIX`` variable should be explicitly set to an empty value.
 
 
 .. _finch: https://github.com/bird-house/finch

birdhouse/components/canarie-api/docker_configuration.py.template

@@ -108,8 +108,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.7.3',
-            'releaseTime': '2025-01-17T18:54:34Z',
+            'version': '2.8.0',
+            'releaseTime': '2025-01-17T23:17:16Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',
@@ -141,8 +141,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.7.3',
-            'releaseTime': '2025-01-17T18:54:34Z',
+            'version': '2.8.0',
+            'releaseTime': '2025-01-17T23:17:16Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',

birdhouse/components/weaver/config/magpie/config.yml.template

@@ -109,6 +109,13 @@ permissions:
     group: anonymous
     action: create
 
+  # HTML rendering files
+  - service: ${WEAVER_MANAGER_NAME}
+    resource: /static
+    permission: read
+    group: anonymous
+    action: create
+
   # Process deployment (write) and listing (read)
   # use 'read-match' to allow only listing, and not describe underlying processes (require 'read' on them individually)
   - service: ${WEAVER_MANAGER_NAME}

birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template

@@ -1,25 +1,51 @@
 
+    location = /weaver-auth {
+        internal;
+        # note: using 'TWITCHER_VERIFY_PATH' path to avoid performing the request via proxy 'TWITCHER_PROTECTED_PATH'
+        # This ensures that access is validated for the user, but does not trigger its access/download twice.
+        # It is also more efficient, since less contents are transferred/buffered.
+        proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/$request_uri;
+        proxy_pass_request_body off;
+        proxy_set_header Host $host;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+    }
+
+    location = /${WEAVER_MANAGER_NAME} {
+        return 301 /${WEAVER_MANAGER_NAME}/$is_args$args;
+    }
+    location ~ ^/${WEAVER_MANAGER_NAME}/(.*)$ {
+        auth_request /weaver-auth;
+        auth_request_set $auth_status $upstream_status;
+
+        # NOTE:
+        #   Inject the 'WEAVER_MANAGER_NAME' prefix here to align with 'SCRIPT_NAME' in the docker-compose config.
+        #   This is needed to help UI elements resolve the full URI path with proxy service prefixes since the
+        #   generated locations returned that must be interpreted/retrieved by the client/browser would otherwise
+        #   not be aware of the proxy redirection path prefix, leading to unresolved resources.
+        proxy_pass http://weaver:4001/${WEAVER_MANAGER_NAME}/$1$is_args$args;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-Host $http_host:$server_port;
+        proxy_buffering off;
+    }
+
     # NOTE:
     #   Redirect to internal network of twitcher with Weaver root endpoint and alias allows to set
     #   the same 'magpie' permissions on the 'weaver' service defined by "WEAVER_MANAGER_NAME".
     #   This allows verification of the same service user/group permissions references regardless
     #   whether the *shortcut* Weaver endpoint, the alias or the explicit 'twitcher' proxy route is used.
-    # redirect EMS/ADES to actual secured Weaver path
-    #location /${WEAVER_CONFIG} {
-    #    return 302 ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME};
-    #}
-
-    location /${WEAVER_MANAGER_NAME} {
-        proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME};
-        proxy_set_header Host $host;
-        proxy_buffering off;
-        include /etc/nginx/conf.d/cors.include;
+    location = ${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME} {
+        return 301 /${WEAVER_MANAGER_NAME}/$is_args$args;
+    }
+    location ~ ^${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}/(.*)$ {
+        return 308 /${WEAVER_MANAGER_NAME}/$1$is_args$args;
     }
 
-    # NOTE:
-    #   this is needed only if not using the location already provided by the core configuration
-    #   see 'birdhouse/components/proxy/conf.d/all-services.include.template'
-    # location where process job outputs will be accessible
-    #location ^~ ${WEAVER_WPS_OUTPUTS_PATH}/ {
-    #    alias ${WEAVER_WPS_OUTPUTS_DIR}/;
-    #}
+    # optional alternate endpoint to access weaver (see 'components/weaver/default.env')
+    ${WEAVER_ALT_PREFIX_PROXY_LOCATION}

birdhouse/components/weaver/default.env

@@ -29,13 +29,15 @@ EXTRA_VARS='
   $WEAVER_WPS_PROVIDERS_MAX_TIME
   $WEAVER_WPS_PROVIDERS_RETRY_COUNT
   $WEAVER_WPS_PROVIDERS_RETRY_AFTER
+  $WEAVER_ALT_PREFIX_PROXY_LOCATION
 '
 # extend the original 'VARS' from 'birdhouse/birdhouse-compose.sh' to employ them for template substitution
 # adding them to 'VARS', they will also be validated in case of override of 'default.env' using 'env.local'
 VARS="$VARS $EXTRA_VARS"
 
 OPTIONAL_VARS="
   $OPTIONAL_VARS
+  \$WEAVER_ALT_PREFIX
   \$WEAVER_DOCKER
   \$WEAVER_VERSION
   \$WEAVER_WORKER_IMAGE
@@ -53,7 +55,7 @@ OPTIONAL_VARS="
 export WEAVER_CONFIG=HYBRID
 
 # default release version that will be used to fetch docker images (API mananger & celery workers services)
-export WEAVER_VERSION=5.6.1
+export WEAVER_VERSION=6.1.1
 export WEAVER_DOCKER=pavics/weaver
 export WEAVER_IMAGE='${WEAVER_DOCKER}:${WEAVER_VERSION}'
 export WEAVER_MANAGER_IMAGE='${WEAVER_IMAGE}-manager'
@@ -63,7 +65,8 @@ export WEAVER_IMAGE_URI='registry.hub.docker.com/${WEAVER_IMAGE}'
 # default release of the MongoDB version employed by Weaver
 # NOTE:
 #   MongoDB>=5.0 is REQUIRED for Weaver>=4.5.0
-export WEAVER_MONGODB_VERSION=5.0.4
+#   MongoDB==7.x works, but default remains 5.0 to avoid DB migration issues (update manually as desired)
+export WEAVER_MONGODB_VERSION=5.0
 # URL is used by both Weaver API and Celery Worker
 # it should contain the docker service name as host to map using shared link between images
 # if credentials are desired, they can be defined with the override of the URL variable
@@ -96,6 +99,17 @@ export WEAVER_WPS_OUTPUTS_PATH="/wpsoutputs/weaver"
 export WEAVER_WPS_OUTPUTS_DIR='${BIRDHOUSE_WPS_OUTPUTS_DIR}/weaver'
 export WEAVER_WPS_WORKDIR="/tmp/wps_workdir/weaver"
 
+# Optional alternate endpoint that will redirect to Weaver.
+# If explicitly set to empty value, it will not be configured in the proxy.
+export WEAVER_ALT_PREFIX=/ogcapi
+export WEAVER_ALT_PREFIX_PROXY_LOCATION='
+    $([ -z "${WEAVER_ALT_PREFIX}" ] && echo "" || echo "
+    location ~ ^${WEAVER_ALT_PREFIX}(.*)\$ {
+        return 301 /${WEAVER_MANAGER_NAME}\$1\$is_args\$args;
+    }
+")
+'
+
 # logging
 export WEAVER_MANAGER_LOG_LEVEL=INFO
 export WEAVER_WORKER_LOG_LEVEL=INFO
@@ -116,6 +130,7 @@ export WEAVER_UNREGISTER_DROPPED_PROVIDERS="False"
 
 export DELAYED_EVAL="
   $DELAYED_EVAL
+  WEAVER_ALT_PREFIX_PROXY_LOCATION
   WEAVER_WPS_OUTPUTS_DIR
   WEAVER_MONGODB_DATA_DIR
   WEAVER_MONGODB_URL
@@ -124,3 +139,10 @@ export DELAYED_EVAL="
   WEAVER_MANAGER_IMAGE
   WEAVER_WORKER_IMAGE
 "
+
+COMPONENT_DEPENDENCIES="
+  $COMPONENT_DEPENDENCIES
+  ./components/wps_outputs-volume
+  ./components/magpie
+  ./components/twitcher
+"

birdhouse/components/weaver/docker-compose-extra.yml

@@ -19,6 +19,10 @@ services:
     image: ${WEAVER_MANAGER_IMAGE}
     environment:
       HOSTNAME: ${BIRDHOUSE_FQDN}
+      # 'HTTP_HOST' and 'SCRIPT_NAME' are used to guide pyramid in the resolution of resources, such as
+      # when invoking the 'static_url' endpoint, so it can be made aware of reverse-proxy context
+      HTTP_HOST: ${BIRDHOUSE_FQDN}
+      SCRIPT_NAME: /${WEAVER_MANAGER_NAME}
       FORWARDED_ALLOW_IPS: "*"
     #env_file:
     #  - ./components/mongodb/credentials.env

docs/source/conf.py

@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.7.3'
+version = '2.8.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.7.3'
+release = '2.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.