diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index 474a00072..c00780e2e 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -20,7 +20,7 @@ postgresql_support_repository_channel: "main" postgresql_support_libpq_version: "13.*.pgdg20.04+1" postgresql_support_psycopg2_version: "2.8.*" postgis_version: "3" -postgis_package_version: "3.3*pgdg20.04+1" +postgis_package_version: "3.4*pgdg20.04+1" daemontools_version: "1:0.76-7" @@ -44,9 +44,10 @@ docker_compose_version: "1.26.*" geop_host: "localhost" geop_port: 8090 -geop_version: "5.3.0" +geop_version: "5.4.0" geop_cache_enabled: 1 -geop_timeout: 200 +geop_timeout: 200s +geop_bucket: datahub-catalogs-us-east-1 nginx_cache_dir: "/var/cache/nginx" diff --git a/deployment/ansible/roles/model-my-watershed.base/defaults/main.yml b/deployment/ansible/roles/model-my-watershed.base/defaults/main.yml index 57a72e8fa..d56ade259 100644 --- a/deployment/ansible/roles/model-my-watershed.base/defaults/main.yml +++ b/deployment/ansible/roles/model-my-watershed.base/defaults/main.yml @@ -14,6 +14,7 @@ envdir_config: MMW_GEOPROCESSING_PORT: "{{ geop_port }}" MMW_GEOPROCESSING_VERSION: "{{ geop_version }}" MMW_GEOPROCESSING_TIMEOUT: "{{ geop_timeout }}" + MMW_GEOPROCESSING_BUCKET: "{{ geop_bucket }}" MMW_ITSI_CLIENT_ID: "{{ itsi_client_id }}" MMW_ITSI_SECRET_KEY: "{{ itsi_secret_key }}" MMW_ITSI_BASE_URL: "{{ itsi_base_url }}" diff --git a/deployment/ansible/roles/model-my-watershed.geoprocessing/templates/systemd-geoprocessing.service.j2 b/deployment/ansible/roles/model-my-watershed.geoprocessing/templates/systemd-geoprocessing.service.j2 index dbd6ec82b..2a93bcab4 100644 --- a/deployment/ansible/roles/model-my-watershed.geoprocessing/templates/systemd-geoprocessing.service.j2 +++ b/deployment/ansible/roles/model-my-watershed.geoprocessing/templates/systemd-geoprocessing.service.j2 @@ -4,13 +4,12 @@ After=network.target [Service] {% if ['development', 'test'] | some_are_in(group_names) -%} -Environment=MMW_GEOPROCESSING_TIMEOUT={{ geop_timeout }}s AWS_PROFILE={{ aws_profile }} -{% else %} -Environment=MMW_GEOPROCESSING_TIMEOUT={{ geop_timeout }}s +Environment=AWS_PROFILE={{ aws_profile }} {% endif %} +Environment="JAVA_WITH_ENV=/usr/bin/envdir /etc/mmw.d/env /usr/bin/java" User=mmw WorkingDirectory={{ geop_home }} -ExecStart=/usr/bin/java -jar mmw-geoprocessing-{{ geop_version }}.jar +ExecStart=/bin/sh -c '${JAVA_WITH_ENV} -jar mmw-geoprocessing-{{ geop_version }}.jar' StandardOutput=syslog StandardError=syslog SyslogIdentifier=geoprocessing diff --git a/scripts/aws/setupdb.sh b/scripts/aws/setupdb.sh index 60c2e26f2..c74014a53 100755 --- a/scripts/aws/setupdb.sh +++ b/scripts/aws/setupdb.sh @@ -15,8 +15,9 @@ where options are one or more of: \n -d load/reload DRB stream data\n -m load/reload mapshed data\n -p load/reload DEP data\n - -c load/reload nhdplus catchment data + -c load/reload nhdplus catchment data\n -q load/reload water quality data\n + -X purge s3 cache as well\n -x purge s3 cache for given path\n " @@ -29,6 +30,7 @@ load_hires_stream=false load_mapshed=false load_water_quality=false load_catchment=false +should_purge_cache=false while getopts ":hbsSdpmqcf:x:" opt; do case $opt in @@ -55,6 +57,8 @@ while getopts ":hbsSdpmqcf:x:" opt; do file_to_load=$OPTARG ;; x) path_to_purge=$OPTARG ;; + X) + should_purge_cache=true ;; \?) echo "invalid option: -$OPTARG" exit ;; @@ -85,9 +89,11 @@ function download_and_load { } function purge_tile_cache { - for path in "${PATHS[@]}"; do - aws s3 rm --recursive "s3://tile-cache.${PUBLIC_HOSTED_ZONE_NAME}/${path}/" - done + if [ "$should_purge_cache" = "true" ] ; then + for path in "${PATHS[@]}"; do + aws s3 rm --recursive "s3://tile-cache.${PUBLIC_HOSTED_ZONE_NAME}/${path}/" + done + fi } function create_trgm_indexes { diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 384965627..3dae62d6f 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -9,6 +9,8 @@ https://docs.djangoproject.com/en/1.7/ref/settings/ """ +import re + from os import environ from os.path import abspath, basename, dirname, join, normpath from sys import path @@ -128,7 +130,11 @@ def get_env_setting(setting): CELERY_TASK_DEFAULT_EXCHANGE = 'tasks' CELERY_TASK_DEFAULT_ROUTING_KEY = "task.%s" % STACK_COLOR -CELERY_TASK_TIME_LIMIT = int(environ.get('MMW_GEOPROCESSING_TIMEOUT', 120)) +# MMW_GEOPROCESSING_TIMEOUT specified with "s" suffix for mmw-geoprocessing +MMW_GEOPROCESSING_TIMEOUT = environ.get('MMW_GEOPROCESSING_TIMEOUT', '120s') + +CELERY_TASK_TIME_LIMIT = int( + re.search(r'\d+', MMW_GEOPROCESSING_TIMEOUT).group()) TASK_REQUEST_TIMEOUT = CELERY_TASK_TIME_LIMIT - 10 # END CELERY CONFIGURATION