diff --git a/.gitignore b/.gitignore index 705b30ebe..897462bbc 100644 --- a/.gitignore +++ b/.gitignore @@ -50,7 +50,7 @@ nosetests.xml coverage.xml # Translations -*.mo +# *.mo # (We DO track these, as we don't want to compile translations for every installation.) *.pot # Django stuff: diff --git a/Dockerfile.mediator b/Dockerfile.mediator index 3b8c33967..cd7281cac 100644 --- a/Dockerfile.mediator +++ b/Dockerfile.mediator @@ -24,11 +24,11 @@ RUN pip3 install -r requirements.txt # TODO: this is just a temporary solution, use pip for production as soon as geometalab.osmaxx is published there ADD ./osmaxx $HOME/osmaxx -ADD ./osmaxx_conversion_service $HOME/osmaxx_conversion_service +ADD ./conversion_service $HOME/conversion_service # Expose modules: ENV PYTHONPATH=PYTHONPATH:$HOME -ENV DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.production +ENV DJANGO_SETTINGS_MODULE=conversion_service.config.settings.production RUN mkdir -p $HOME/docker_entrypoint/osmaxx/conversion_service $HOME/entrypoint COPY ./docker_entrypoint/osmaxx/conversion_service $HOME/entrypoint diff --git a/Dockerfile.worker b/Dockerfile.worker index 7fc61b517..f6d70f97a 100644 --- a/Dockerfile.worker +++ b/Dockerfile.worker @@ -104,10 +104,12 @@ RUN apt-get clean && apt-get update && \ WORKDIR /root/osm2pgsql # OSM2PGSQL +ENV OSM2PGSQL_VERSION=0.92.0 RUN mkdir src &&\ cd src &&\ GIT_SSL_NO_VERIFY=true git clone https://github.com/openstreetmap/osm2pgsql.git &&\ cd osm2pgsql &&\ + git checkout ${OSM2PGSQL_VERSION} &&\ mkdir build &&\ cd build &&\ cmake ..&&\ @@ -166,11 +168,11 @@ RUN pip3 install -r requirements.txt # TODO: this is just a temporary solution, use pip for production as soon as geometalab.osmaxx is published there ADD ./osmaxx $HOME/osmaxx -ADD ./osmaxx_conversion_service $HOME/osmaxx_conversion_service +ADD ./conversion_service $HOME/conversion_service # expose modules ENV PYTHONPATH=PYTHONPATH:$HOME -ENV DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.worker +ENV DJANGO_SETTINGS_MODULE=conversion_service.config.settings.worker ENV WORKER_QUEUES default high ENTRYPOINT ["/home/py/entrypoint/entrypoint.sh"] diff --git a/conversion_service/Procfile.mediator.dev b/conversion_service/Procfile.mediator.dev new file mode 100644 index 000000000..474a6d7c1 --- /dev/null +++ b/conversion_service/Procfile.mediator.dev @@ -0,0 +1,2 @@ +mediator: python3 ./conversion_service/manage.py runserver_plus ${APP_HOST}:${APP_PORT} +harvester: python3 ./conversion_service/manage.py result_harvester diff --git a/conversion_service/Procfile.mediator.prod b/conversion_service/Procfile.mediator.prod new file mode 100644 index 000000000..e7e59ed08 --- /dev/null +++ b/conversion_service/Procfile.mediator.prod @@ -0,0 +1,2 @@ +mediator: gunicorn --workers ${NUM_WORKERS} conversion_service.config.wsgi --bind ${APP_HOST}:${APP_PORT} +harvester: python3 ./conversion_service/manage.py result_harvester diff --git a/conversion_service/Procfile.worker b/conversion_service/Procfile.worker new file mode 100644 index 000000000..77935082c --- /dev/null +++ b/conversion_service/Procfile.worker @@ -0,0 +1 @@ +worker: ${HOME}/entrypoint/wait-for-it.sh localhost:5432 -t 30 && python3 ./conversion_service/manage.py rqworker ${WORKER_QUEUES:-default high} diff --git a/osmaxx/utilities/__init__.py b/conversion_service/config/__init__.py similarity index 100% rename from osmaxx/utilities/__init__.py rename to conversion_service/config/__init__.py diff --git a/osmaxx_conversion_service/config/__init__.py b/conversion_service/config/settings/__init__.py similarity index 100% rename from osmaxx_conversion_service/config/__init__.py rename to conversion_service/config/settings/__init__.py diff --git a/osmaxx_conversion_service/config/settings/common.py b/conversion_service/config/settings/common.py similarity index 98% rename from osmaxx_conversion_service/config/settings/common.py rename to conversion_service/config/settings/common.py index dd757e349..4dc1f8e9b 100644 --- a/osmaxx_conversion_service/config/settings/common.py +++ b/conversion_service/config/settings/common.py @@ -43,7 +43,6 @@ LOCAL_APPS = ( 'osmaxx.version', 'osmaxx.clipping_area', - 'osmaxx.conversion_api', 'osmaxx.conversion', ) @@ -190,10 +189,10 @@ # URL Configuration # ------------------------------------------------------------------------------ -ROOT_URLCONF = 'osmaxx_conversion_service.config.urls' +ROOT_URLCONF = 'conversion_service.config.urls' # See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application -WSGI_APPLICATION = 'osmaxx_conversion_service.config.wsgi.application' +WSGI_APPLICATION = 'conversion_service.config.wsgi.application' CACHES = { 'default': { diff --git a/osmaxx_conversion_service/config/settings/local.py b/conversion_service/config/settings/local.py similarity index 100% rename from osmaxx_conversion_service/config/settings/local.py rename to conversion_service/config/settings/local.py diff --git a/osmaxx_conversion_service/config/settings/production.py b/conversion_service/config/settings/production.py similarity index 100% rename from osmaxx_conversion_service/config/settings/production.py rename to conversion_service/config/settings/production.py diff --git a/osmaxx_conversion_service/config/settings/worker.py b/conversion_service/config/settings/worker.py similarity index 100% rename from osmaxx_conversion_service/config/settings/worker.py rename to conversion_service/config/settings/worker.py diff --git a/osmaxx_conversion_service/config/urls.py b/conversion_service/config/urls.py similarity index 100% rename from osmaxx_conversion_service/config/urls.py rename to conversion_service/config/urls.py diff --git a/osmaxx_conversion_service/config/wsgi.py b/conversion_service/config/wsgi.py similarity index 81% rename from osmaxx_conversion_service/config/wsgi.py rename to conversion_service/config/wsgi.py index cf05ba6a3..5ac4decff 100644 --- a/osmaxx_conversion_service/config/wsgi.py +++ b/conversion_service/config/wsgi.py @@ -12,7 +12,7 @@ from whitenoise.django import DjangoWhiteNoise from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "osmaxx_conversion_service.config.settings.production") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conversion_service.config.settings.production") application = get_wsgi_application() application = Sentry(DjangoWhiteNoise(application)) diff --git a/osmaxx_conversion_service/manage.py b/conversion_service/manage.py similarity index 71% rename from osmaxx_conversion_service/manage.py rename to conversion_service/manage.py index 1b021a6e4..84fa0c294 100755 --- a/osmaxx_conversion_service/manage.py +++ b/conversion_service/manage.py @@ -5,7 +5,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "osmaxx_conversion_service.config.settings.local") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conversion_service.config.settings.local") from django.core.management import execute_from_command_line diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b3462b4e0..46880a2e7 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -13,7 +13,7 @@ services: volumes: - ./osmaxx:/home/py/osmaxx - ./web_frontend:/home/py/web_frontend - - ./osmaxx_conversion_service:/home/py/osmaxx_conversion_service + - ./conversion_service:/home/py/conversion_service ports: - "8000:8000" command: [honcho, -f, ./web_frontend/Procfile.django.dev, start] @@ -42,11 +42,11 @@ services: volumes: - ./osmaxx:/home/py/osmaxx - ./web_frontend:/home/py/web_frontend - - ./osmaxx_conversion_service:/home/py/osmaxx_conversion_service - command: [honcho, -f, ./osmaxx_conversion_service/Procfile.mediator.dev, start] + - ./conversion_service:/home/py/conversion_service + command: [honcho, -f, ./conversion_service/Procfile.mediator.dev, start] environment: - DJANGO_SECRET_KEY=insecure!2 - - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.local + - DJANGO_SETTINGS_MODULE=conversion_service.config.settings.local logging: driver: "json-file" options: @@ -55,7 +55,7 @@ services: volumes: - ./osmaxx:/home/py/osmaxx - ./web_frontend:/home/py/web_frontend - - ./osmaxx_conversion_service:/home/py/osmaxx_conversion_service + - ./conversion_service:/home/py/conversion_service build: context: . dockerfile: Dockerfile.worker @@ -69,7 +69,7 @@ services: volumes: - ./osmaxx:/home/py/osmaxx - ./web_frontend:/home/py/web_frontend - - ./osmaxx_conversion_service:/home/py/osmaxx_conversion_service + - ./conversion_service:/home/py/conversion_service build: context: . dockerfile: Dockerfile.worker diff --git a/docker-compose.yml b/docker-compose.yml index ac734648c..2ea6a329c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,7 +60,7 @@ services: extends: file: docker-compose-common.yml service: conversionbase - command: [honcho, -f, ./osmaxx_conversion_service/Procfile.mediator.prod, start] + command: [honcho, -f, ./conversion_service/Procfile.mediator.prod, start] expose: - "8901" volumes: @@ -77,7 +77,7 @@ services: - REDIS_HOST=redis - REDIS_PORT=6379 # comma separated list, no brackets, e.g. localhost,dev.myhost.com - - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.production + - DJANGO_SETTINGS_MODULE=conversion_service.config.settings.production - NUM_WORKERS=5 depends_on: - conversionserviceredis @@ -89,7 +89,7 @@ services: extends: file: docker-compose-common.yml service: conversionbase - command: [honcho, -f, ./osmaxx_conversion_service/Procfile.worker, start] + command: [honcho, -f, ./conversion_service/Procfile.worker, start] volumes: - osm_data:/var/data/osm-planet - worker-data:/data/media/job_result_files @@ -101,7 +101,7 @@ services: environment: - REDIS_HOST=redis - REDIS_PORT=6379 - - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.worker + - DJANGO_SETTINGS_MODULE=conversion_service.config.settings.worker depends_on: - conversionserviceredis - osmboundaries-database @@ -113,7 +113,7 @@ services: extends: file: docker-compose-common.yml service: conversionbase - command: [honcho, -f, ./osmaxx_conversion_service/Procfile.worker, start] + command: [honcho, -f, ./conversion_service/Procfile.worker, start] volumes: - osm_data:/var/data/osm-planet - worker-data:/data/media/job_result_files @@ -125,7 +125,7 @@ services: environment: - REDIS_HOST=redis - REDIS_PORT=6379 - - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.worker + - DJANGO_SETTINGS_MODULE=conversion_service.config.settings.worker # only listens for jobs in this specific queue - WORKER_QUEUES=high depends_on: diff --git a/docker_entrypoint/osmaxx/conversion_service/entrypoint.sh b/docker_entrypoint/osmaxx/conversion_service/entrypoint.sh index d2dd08319..d3a174d1a 100755 --- a/docker_entrypoint/osmaxx/conversion_service/entrypoint.sh +++ b/docker_entrypoint/osmaxx/conversion_service/entrypoint.sh @@ -4,7 +4,7 @@ set -e # wait for at most 30s for the db to be up ${HOME}/entrypoint/wait-for-it.sh ${DATABASE_HOST}:${DATABASE_PORT} -t 30 -python3 osmaxx_conversion_service/manage.py migrate --no-input && \ -python3 osmaxx_conversion_service/manage.py collectstatic --noinput && \ +python3 conversion_service/manage.py migrate --no-input && \ +python3 conversion_service/manage.py collectstatic --noinput && \ python3 entrypoint/create_user_entrypoint.py && \ exec "$@" diff --git a/docs/development/releasing.md b/docs/development/releasing.md index 98820cbdd..276b0eacd 100644 --- a/docs/development/releasing.md +++ b/docs/development/releasing.md @@ -22,7 +22,7 @@ The steps execute in the script in a nutshell are: - `git flow release publish ` - `adapt in osmaxx/__init__.py` - `python ./web_frontend/manage.py makemessages -l de_CH -l en -l en_UK -l en_US` -- `python ./osmaxx_conversion_service/manage.py makemessages -l de_CH -l en -l en_UK -l en_US` +- `python ./conversion_service/manage.py makemessages -l de_CH -l en -l en_UK -l en_US` - `git flow release finish ` - `git push` - `git push --tags` diff --git a/osmaxx/__init__.py b/osmaxx/__init__.py index 9e9401fc8..40d84aee8 100644 --- a/osmaxx/__init__.py +++ b/osmaxx/__init__.py @@ -1,4 +1,4 @@ -__version__ = 'v3.5.0' +__version__ = 'v3.10.0' __all__ = [ '__version__', diff --git a/osmaxx/contrib/auth/frontend_permissions.py b/osmaxx/contrib/auth/frontend_permissions.py index 46db2bace..b652a46d7 100644 --- a/osmaxx/contrib/auth/frontend_permissions.py +++ b/osmaxx/contrib/auth/frontend_permissions.py @@ -1,4 +1,4 @@ -from django.contrib.auth.decorators import login_required, user_passes_test +from django.contrib.auth.decorators import user_passes_test from django.core.urlresolvers import reverse_lazy from django.utils.decorators import method_decorator from rest_framework import permissions @@ -6,14 +6,6 @@ from osmaxx.profile.models import Profile -def _may_user_access_osmaxx_frontend(user): - """ - Actual test to check if the user is in the frontend user group, - to give access or deny it. Note: Admins have superpowers. - """ - return user.has_perm('excerptexport.add_extractionorder') - - def _may_user_access_this_excerpt(user, excerpt): return excerpt.is_public or excerpt.owner == user @@ -30,21 +22,6 @@ def _user_has_validated_email(user): return profile.has_validated_email() -def frontend_access_required(function=None): - """ - Decorator for views that checks that the user has the correct access rights, - redirecting to the information page if necessary. - """ - access_denied_info_url = reverse_lazy('excerptexport:access_denied') - actual_decorator = user_passes_test( - _may_user_access_osmaxx_frontend, - login_url=access_denied_info_url - ) - if function: - return actual_decorator(function) - return actual_decorator - - def validated_email_required(function=None): """ Decorator for views that checks that the user has set a validated email, @@ -60,24 +37,6 @@ def validated_email_required(function=None): return actual_decorator -class LoginRequiredMixin(object): - """ - Login required Mixin for Class Based Views. - """ - @method_decorator(login_required) - def dispatch(self, *args, **kwargs): - return super(LoginRequiredMixin, self).dispatch(*args, **kwargs) - - -class FrontendAccessRequiredMixin(object): - """ - Frontend Access Check Mixin for Class Based Views. - """ - @method_decorator(frontend_access_required) - def dispatch(self, *args, **kwargs): - return super(FrontendAccessRequiredMixin, self).dispatch(*args, **kwargs) - - class EmailRequiredMixin(object): """ Frontend Access Check Mixin for Class Based Views. @@ -89,11 +48,11 @@ def dispatch(self, *args, **kwargs): class AuthenticatedAndAccessPermission(permissions.BasePermission): """ - Allows access only to authenticated users with frontend permissions. + Allows access only to authenticated users with confirmed email address. """ def has_permission(self, request, view): - return request.user.is_authenticated and _may_user_access_osmaxx_frontend(request.user) + return request.user.is_authenticated and _user_has_validated_email(request.user) class HasBBoxAccessPermission(permissions.BasePermission): diff --git a/osmaxx/conversion/__init__.py b/osmaxx/conversion/__init__.py index cbb44626e..df64b7849 100644 --- a/osmaxx/conversion/__init__.py +++ b/osmaxx/conversion/__init__.py @@ -1 +1,5 @@ +from .constants import output_format + default_app_config = 'osmaxx.conversion.apps.ConversionConfig' + +__all__ = ['default_app_config', 'output_format'] diff --git a/osmaxx_conversion_service/config/settings/__init__.py b/osmaxx/conversion/constants/__init__.py similarity index 100% rename from osmaxx_conversion_service/config/settings/__init__.py rename to osmaxx/conversion/constants/__init__.py diff --git a/osmaxx/conversion_api/coordinate_reference_systems.py b/osmaxx/conversion/constants/coordinate_reference_systems.py similarity index 100% rename from osmaxx/conversion_api/coordinate_reference_systems.py rename to osmaxx/conversion/constants/coordinate_reference_systems.py diff --git a/osmaxx/conversion_api/formats.py b/osmaxx/conversion/constants/output_format.py similarity index 60% rename from osmaxx/conversion_api/formats.py rename to osmaxx/conversion/constants/output_format.py index 92d01b69c..b3daafe35 100644 --- a/osmaxx/conversion_api/formats.py +++ b/osmaxx/conversion/constants/output_format.py @@ -1,62 +1,45 @@ import uuid -from collections import OrderedDict +from collections import OrderedDict, namedtuple -from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy as _ FGDB, SHAPEFILE, GPKG, SPATIALITE, GARMIN, PBF = 'fgdb', 'shapefile', 'gpkg', 'spatialite', 'garmin', 'pbf' -class OutputFormat: - def __init__( - self, *, long_identifier, verbose_name, archive_file_name_identifier, abbreviations, is_white_box, - layer_filename_extension=None - ): - self._long_identifier = long_identifier - self._verbose_name = verbose_name - self._archive_file_name_identifier = archive_file_name_identifier - self._abbreviations = abbreviations - self._is_white_box = is_white_box - self._layer_filename_extension = layer_filename_extension +_OutputFormatBase = namedtuple( + 'OutputFormatBase', + [ + 'long_identifier', + 'verbose_name', + 'archive_file_name_identifier', + 'abbreviations', + 'is_white_box', + 'layer_filename_extension', + ] +) +_OutputFormatBase.__new__.__defaults__ = (None,) # This makes the last field (i.e. layer_filename_extension) optional. - @property - def long_identifier(self): - return self._long_identifier - - @property - def verbose_name(self): - return self._verbose_name - - @property - def archive_file_name_identifier(self): - return self._archive_file_name_identifier - - @property - def abbreviations(self): - return self._abbreviations - - @property - def layer_filename_extension(self): - return self._layer_filename_extension +class OutputFormat(_OutputFormatBase): @property def qgis_datasource_separator(self): """ The string used to separate the dataset path (collections of layers) from the individual layer in a QGIS project file's ```` element referring to data in this format. """ - return '/' if self._layer_filename_extension is not None else '|layername=' + return '/' if self.layer_filename_extension is not None else '|layername=' def unique_archive_name(self): return "{}_{}.zip".format(uuid.uuid4(), self.archive_file_name_identifier) def crs_change_available(self): - return self._is_white_box + return self.is_white_box def detail_level_available(self): - return self._is_white_box + return self.is_white_box -FORMAT_DEFINITIONS = OrderedDict([ +DEFINITIONS = OrderedDict([ (FGDB, OutputFormat( long_identifier='Esri File Geodatabase', verbose_name=_('Esri File Geodatabase'), @@ -102,4 +85,5 @@ def detail_level_available(self): )), ]) -FORMAT_CHOICES = tuple((key, definition.verbose_name) for key, definition in FORMAT_DEFINITIONS.items()) +CHOICES = tuple((key, definition.verbose_name) for key, definition in DEFINITIONS.items()) +ALL = DEFINITIONS.keys() diff --git a/osmaxx/conversion_api/statuses.py b/osmaxx/conversion/constants/statuses.py similarity index 100% rename from osmaxx/conversion_api/statuses.py rename to osmaxx/conversion/constants/statuses.py diff --git a/osmaxx/conversion/converters/converter.py b/osmaxx/conversion/converters/converter.py index d7b433c62..b59f0588b 100644 --- a/osmaxx/conversion/converters/converter.py +++ b/osmaxx/conversion/converters/converter.py @@ -1,68 +1,20 @@ -from osmaxx.conversion.converters.converter_garmin.garmin import Garmin -from osmaxx.conversion.converters.converter_gis.gis import GISConverter -from osmaxx.conversion.converters.converter_pbf.to_pbf import produce_pbf +from osmaxx.conversion import output_format +from osmaxx.conversion.converters import converter_garmin +from osmaxx.conversion.converters import converter_gis +from osmaxx.conversion.converters import converter_pbf from osmaxx.conversion.job_dispatcher.rq_dispatcher import rq_enqueue_with_settings -from osmaxx.conversion_api.formats import FGDB, SHAPEFILE, GPKG, SPATIALITE, GARMIN, PBF +from osmaxx.utils.frozendict import frozendict - -class Conversion(object): - def __init__( - self, - *, - conversion_format, - area_name, - osmosis_polygon_file_string, - output_zip_file_path, - filename_prefix, - detail_level, - out_srs=None - ): - self._conversion_format = conversion_format - self._output_zip_file_path = output_zip_file_path - self._area_name = area_name - self._polyfile_string = osmosis_polygon_file_string - self._name_prefix = filename_prefix - self._out_srs = out_srs - self._detail_level = detail_level - - _format_process = { - GARMIN: self._create_garmin_export, - PBF: self._create_pbf, - FGDB: self._extract_postgis_format, - SHAPEFILE: self._extract_postgis_format, - GPKG: self._extract_postgis_format, - SPATIALITE: self._extract_postgis_format, - } - self._conversion_process = _format_process[conversion_format] - - def start_format_extraction(self): - self._conversion_process() - - def _extract_postgis_format(self): - gis = GISConverter( - conversion_format=self._conversion_format, - out_zip_file_path=self._output_zip_file_path, - base_file_name=self._name_prefix, - out_srs=self._out_srs, - polyfile_string=self._polyfile_string, - detail_level=self._detail_level - ) - gis.create_gis_export() - - def _create_garmin_export(self): - garmin = Garmin( - out_zip_file_path=self._output_zip_file_path, - area_name=self._area_name, - polyfile_string=self._polyfile_string, - ) - garmin.create_garmin_export() - - def _create_pbf(self): - produce_pbf( - out_zip_file_path=self._output_zip_file_path, - area_name=self._area_name, - polyfile_string=self._polyfile_string, - ) +_format_converter = frozendict( + { + output_format.GARMIN: converter_garmin, + output_format.PBF: converter_pbf, + output_format.FGDB: converter_gis, + output_format.SHAPEFILE: converter_gis, + output_format.GPKG: converter_gis, + output_format.SPATIALITE: converter_gis, + } +) def convert( @@ -87,6 +39,6 @@ def convert( queue_name=queue_name, **params ).id - conversion = Conversion(**params) - conversion.start_format_extraction() + converter = _format_converter[conversion_format] + converter.perform_export(**params) return None diff --git a/osmaxx/conversion/converters/converter_garmin/__init__.py b/osmaxx/conversion/converters/converter_garmin/__init__.py index e69de29bb..6076ff857 100644 --- a/osmaxx/conversion/converters/converter_garmin/__init__.py +++ b/osmaxx/conversion/converters/converter_garmin/__init__.py @@ -0,0 +1,3 @@ +from .garmin import perform_export + +__all__ = ['perform_export'] diff --git a/osmaxx/conversion/converters/converter_garmin/garmin.py b/osmaxx/conversion/converters/converter_garmin/garmin.py index 052e7ea66..9d1690c17 100644 --- a/osmaxx/conversion/converters/converter_garmin/garmin.py +++ b/osmaxx/conversion/converters/converter_garmin/garmin.py @@ -1,5 +1,4 @@ import shutil -import subprocess import os import tempfile @@ -7,8 +6,19 @@ from rq import get_current_job from osmaxx.conversion._settings import CONVERSION_SETTINGS, odb_license, copying_notice, creative_commons_license +from osmaxx.conversion.converters.converter_pbf.to_pbf import cut_pbf_along_polyfile + +from osmaxx.conversion.converters.utils import zip_folders_relative, recursive_getsize, logged_check_call + + +def perform_export(*, output_zip_file_path, area_name, osmosis_polygon_file_string, **__): + garmin = Garmin( + output_zip_file_path=output_zip_file_path, + area_name=area_name, + polyfile_string=osmosis_polygon_file_string, + ) + garmin.create_garmin_export() -from osmaxx.conversion.converters.utils import zip_folders_relative, recursive_getsize _path_to_commandline_utils = os.path.join(os.path.dirname(__file__), 'command_line_utils') _path_to_bounds_zip = os.path.join(CONVERSION_SETTINGS['SEA_AND_BOUNDS_ZIP_DIRECTORY'], 'bounds.zip') @@ -17,8 +27,8 @@ class Garmin: - def __init__(self, *, out_zip_file_path, area_name, polyfile_string): - self._resulting_zip_file_path = out_zip_file_path + def __init__(self, *, output_zip_file_path, area_name, polyfile_string): + self._resulting_zip_file_path = output_zip_file_path self._map_description = area_name self._osmosis_polygon_file = tempfile.NamedTemporaryFile(suffix='.poly', mode='w') self._osmosis_polygon_file.write(polyfile_string) @@ -26,6 +36,7 @@ def __init__(self, *, out_zip_file_path, area_name, polyfile_string): self._polyfile_path = self._osmosis_polygon_file.name self._start_time = None self._unzipped_result_size = None + self._area_polyfile_string = polyfile_string def create_garmin_export(self): self._start_time = timezone.now() @@ -47,7 +58,9 @@ def _to_garmin(self): def _split(self, workdir): memory_option = '-Xmx7000m' _splitter_path = os.path.abspath(os.path.join(_path_to_commandline_utils, 'splitter', 'splitter.jar')) - subprocess.check_call([ + _pbf_file_path = os.path.join('/tmp', 'pbf_cutted.pbf') + cut_pbf_along_polyfile(self._area_polyfile_string, _pbf_file_path) + logged_check_call([ 'java', memory_option, '-jar', _splitter_path, @@ -55,7 +68,7 @@ def _split(self, workdir): '--description={0}'.format(self._map_description), '--geonames-file={0}'.format(_path_to_geonames_zip), '--polygon-file={}'.format(self._polyfile_path), - CONVERSION_SETTINGS.get('PBF_PLANET_FILE_PATH'), + _pbf_file_path, ]) config_file_path = os.path.join(workdir, 'template.args') return config_file_path @@ -79,7 +92,7 @@ def _produce_garmin(self, config_file_path, out_dir): '--route', ] - subprocess.check_call( + logged_check_call( mkg_map_command + output_dir + config diff --git a/osmaxx/conversion/converters/converter_gis/__init__.py b/osmaxx/conversion/converters/converter_gis/__init__.py index e69de29bb..b2b7a6664 100644 --- a/osmaxx/conversion/converters/converter_gis/__init__.py +++ b/osmaxx/conversion/converters/converter_gis/__init__.py @@ -0,0 +1,3 @@ +from .gis import perform_export + +__all__ = ['perform_export'] diff --git a/osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py b/osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py index 9ab03810d..951d11de4 100644 --- a/osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py +++ b/osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py @@ -1,17 +1,18 @@ import glob import os -import subprocess +from memoize import mproperty + +from osmaxx.conversion.converters.converter_gis.detail_levels import DETAIL_LEVEL_ALL, DETAIL_LEVEL_TABLES from osmaxx.conversion.converters.converter_gis.helper.default_postgres import get_default_postgres_wrapper from osmaxx.conversion.converters.converter_gis.helper.osm_boundaries_importer import OSMBoundariesImporter -from osmaxx.conversion.converters import detail_levels -from osmaxx.conversion.converters.converter_pbf.to_pbf import polyfile_string_to_pbf -from osmaxx.conversion.converters.detail_levels import DETAIL_LEVEL_TABLES +from osmaxx.conversion.converters.converter_pbf.to_pbf import cut_pbf_along_polyfile +from osmaxx.conversion.converters.utils import logged_check_call from osmaxx.utils import polyfile_helpers class BootStrapper: - def __init__(self, area_polyfile_string, *, detail_level=detail_levels.DETAIL_LEVEL_ALL): + def __init__(self, area_polyfile_string, *, detail_level=DETAIL_LEVEL_ALL): self.area_polyfile_string = area_polyfile_string self._postgres = get_default_postgres_wrapper() self._script_base_dir = os.path.abspath(os.path.dirname(__file__)) @@ -22,7 +23,7 @@ def __init__(self, area_polyfile_string, *, detail_level=detail_levels.DETAIL_LE def bootstrap(self): self._reset_database() - polyfile_string_to_pbf(self.area_polyfile_string, self._pbf_file_path) + cut_pbf_along_polyfile(self.area_polyfile_string, self._pbf_file_path) self._import_boundaries() self._import_pbf() self._setup_db_functions() @@ -30,7 +31,7 @@ def bootstrap(self): self._filter_data() self._create_views() - @property + @mproperty def geom(self): return polyfile_helpers.parse_poly_string(self.area_polyfile_string) @@ -129,4 +130,4 @@ def _import_pbf(self): '--input-reader', 'pbf', self._pbf_file_path, ] - subprocess.check_call(osm_2_pgsql_command) + logged_check_call(osm_2_pgsql_command) diff --git a/osmaxx/conversion/converters/detail_levels.py b/osmaxx/conversion/converters/converter_gis/detail_levels.py similarity index 100% rename from osmaxx/conversion/converters/detail_levels.py rename to osmaxx/conversion/converters/converter_gis/detail_levels.py diff --git a/osmaxx/conversion/converters/converter_gis/extract/db_to_format/extract.py b/osmaxx/conversion/converters/converter_gis/extract/db_to_format/extract.py index 66643571e..065691bcc 100644 --- a/osmaxx/conversion/converters/converter_gis/extract/db_to_format/extract.py +++ b/osmaxx/conversion/converters/converter_gis/extract/db_to_format/extract.py @@ -2,25 +2,25 @@ import subprocess from osmaxx.conversion._settings import CONVERSION_SETTINGS -from osmaxx.conversion_api.formats import FGDB, SHAPEFILE, GPKG, SPATIALITE +from osmaxx.conversion import output_format FORMATS = { - FGDB: { + output_format.FGDB: { 'ogr_name': 'FileGDB', 'extension': '.gdb', 'extraction_options': [], }, - GPKG: { + output_format.GPKG: { 'ogr_name': 'GPKG', 'extension': '.gpkg', 'extraction_options': [], }, - SHAPEFILE: { + output_format.SHAPEFILE: { 'ogr_name': 'Esri Shapefile', 'extension': '.shp', 'extraction_options': ['-lco', 'ENCODING=UTF-8'], }, - SPATIALITE: { + output_format.SPATIALITE: { 'ogr_name': 'SQLite', 'extension': '.sqlite', 'extraction_options': ['-dsco', 'SPATIALITE=YES', '-nlt', 'GEOMETRY'] # FIXME: Remove or change -nlt because of geometry reading problems diff --git a/osmaxx/conversion/converters/converter_gis/gis.py b/osmaxx/conversion/converters/converter_gis/gis.py index c29d0ec29..628112648 100644 --- a/osmaxx/conversion/converters/converter_gis/gis.py +++ b/osmaxx/conversion/converters/converter_gis/gis.py @@ -11,11 +11,25 @@ from jinja2 import Environment, PackageLoader from rq import get_current_job +from osmaxx.conversion import output_format from osmaxx.conversion._settings import odb_license from osmaxx.conversion.converters.converter_gis.bootstrap import BootStrapper from osmaxx.conversion.converters.converter_gis.extract.db_to_format.extract import extract_to from osmaxx.conversion.converters.utils import zip_folders_relative, recursive_getsize -from osmaxx.conversion_api.formats import FORMAT_DEFINITIONS + + +def perform_export( + *, conversion_format, output_zip_file_path, filename_prefix, out_srs, osmosis_polygon_file_string, detail_level, + **__): + gis = GISConverter( + conversion_format=conversion_format, + output_zip_file_path=output_zip_file_path, + base_file_name=filename_prefix, + out_srs=out_srs, + polyfile_string=osmosis_polygon_file_string, + detail_level=detail_level + ) + gis.create_gis_export() QGIS_DISPLAY_SRID = 3857 # Web Mercator @@ -28,12 +42,13 @@ class ScaleLevel(Enum): class GISConverter: - def __init__(self, *, conversion_format, out_zip_file_path, base_file_name, out_srs, polyfile_string, detail_level): + def __init__( + self, *, conversion_format, output_zip_file_path, base_file_name, out_srs, polyfile_string, detail_level): """ Converts a specified pbf into the specified format. Args: - out_zip_file_path: path to where the zipped result should be stored, directory must already exist + output_zip_file_path: path to where the zipped result should be stored, directory must already exist conversion_format: One of 'fgdb', 'shapefile', 'gpkg', 'spatialite' base_file_name: base for created files inside the zip file @@ -41,7 +56,7 @@ def __init__(self, *, conversion_format, out_zip_file_path, base_file_name, out_ the path to the resulting zip file """ self._base_file_name = base_file_name - self._out_zip_file_path = out_zip_file_path + self._out_zip_file_path = output_zip_file_path self._polyfile_string = polyfile_string self._conversion_format = conversion_format self._out_srs = out_srs @@ -98,7 +113,7 @@ def _dump_qgis_symbology(self, data_location, geom_in_qgis_display_srs, target_d shutil.copy(qgis_symbology_readme, qgis_symbology_dir) for scale_level in ScaleLevel: template = self._env.get_template('OSMaxx_{}.qgs.jinja2'.format(scale_level.name.upper())) - format_definition = FORMAT_DEFINITIONS[self._conversion_format] + format_definition = output_format.DEFINITIONS[self._conversion_format] template.stream( data_location=os.path.basename(data_location), separator=format_definition.qgis_datasource_separator, diff --git a/osmaxx/conversion/converters/converter_pbf/__init__.py b/osmaxx/conversion/converters/converter_pbf/__init__.py index e69de29bb..4818980c4 100644 --- a/osmaxx/conversion/converters/converter_pbf/__init__.py +++ b/osmaxx/conversion/converters/converter_pbf/__init__.py @@ -0,0 +1,3 @@ +from .to_pbf import produce_pbf as perform_export + +__all__ = ['perform_export'] diff --git a/osmaxx/conversion/converters/converter_pbf/to_pbf.py b/osmaxx/conversion/converters/converter_pbf/to_pbf.py index 563bc149c..dea0bd45b 100644 --- a/osmaxx/conversion/converters/converter_pbf/to_pbf.py +++ b/osmaxx/conversion/converters/converter_pbf/to_pbf.py @@ -1,14 +1,13 @@ import os import shutil -import subprocess import tempfile from django.utils import timezone from rq import get_current_job -from osmaxx.conversion._settings import CONVERSION_SETTINGS, odb_license, copying_notice, creative_commons_license -from osmaxx.conversion.converters.utils import zip_folders_relative, recursive_getsize +from osmaxx.conversion._settings import CONVERSION_SETTINGS, odb_license +from osmaxx.conversion.converters.utils import zip_folders_relative, recursive_getsize, logged_check_call def cut_area_from_pbf(pbf_result_file_path, extent_polyfile_path): @@ -21,33 +20,32 @@ def cut_area_from_pbf(pbf_result_file_path, extent_polyfile_path): "-B={}".format(extent_polyfile_path), "{}".format(CONVERSION_SETTINGS["PBF_PLANET_FILE_PATH"]), ] - subprocess.check_call(command) + logged_check_call(command) -def polyfile_string_to_pbf(polyfile_string, pbf_out_path): - polyfile_path = os.path.join('/tmp', 'polyfile_extent.poly') - with open(polyfile_path, 'w') as f: - f.write(polyfile_string) - cut_area_from_pbf(pbf_out_path, polyfile_path) +def cut_pbf_along_polyfile(polyfile_string, pbf_out_path): + with tempfile.NamedTemporaryFile('w') as polyfile: + polyfile.write(polyfile_string) + polyfile.flush() + os.fsync(polyfile) + cut_area_from_pbf(pbf_out_path, polyfile.name) -def produce_pbf(*, out_zip_file_path, area_name, polyfile_string): +def produce_pbf(*, output_zip_file_path, filename_prefix, osmosis_polygon_file_string, **__): _start_time = timezone.now() with tempfile.TemporaryDirectory() as tmp_dir: out_dir = os.path.join(tmp_dir, 'pbf') os.makedirs(out_dir, exist_ok=True) - pbf_out_path = os.path.join(out_dir, area_name + '.pbf') + pbf_out_path = os.path.join(out_dir, filename_prefix + '.pbf') - shutil.copy(copying_notice, out_dir) shutil.copy(odb_license, out_dir) - shutil.copy(creative_commons_license, out_dir) - polyfile_string_to_pbf(polyfile_string, pbf_out_path) + cut_pbf_along_polyfile(osmosis_polygon_file_string, pbf_out_path) unzipped_result_size = recursive_getsize(out_dir) - zip_folders_relative([tmp_dir], out_zip_file_path) + zip_folders_relative([tmp_dir], output_zip_file_path) job = get_current_job() if job: diff --git a/osmaxx/conversion/converters/utils.py b/osmaxx/conversion/converters/utils.py index ec79c7ea4..64e1d24dc 100644 --- a/osmaxx/conversion/converters/utils.py +++ b/osmaxx/conversion/converters/utils.py @@ -1,8 +1,9 @@ +import logging +import os +import subprocess import uuid import zipfile -import os - # Use the built-in version of scandir if possible, otherwise # use the scandir module version try: @@ -10,6 +11,8 @@ except ImportError: from scandir import scandir +logger = logging.getLogger(__name__) + def zip_folders_relative(folder_list, zip_out_file_path=None): """ @@ -42,3 +45,11 @@ def recursive_getsize(path): elif entry.is_dir(follow_symlinks=False): size += recursive_getsize(os.path.join(path, entry.path)) return size + + +def logged_check_call(*args, **kwargs): + try: + subprocess.check_call(*args, **kwargs) + except subprocess.CalledProcessError as e: + logger.error('Command `{}` exited with return value {}\nOutput:\n{}'.format(e.cmd, e.returncode, e.output)) + raise diff --git a/osmaxx/conversion/management/commands/result_harvester.py b/osmaxx/conversion/management/commands/result_harvester.py index a40470f0a..59997ab06 100644 --- a/osmaxx/conversion/management/commands/result_harvester.py +++ b/osmaxx/conversion/management/commands/result_harvester.py @@ -12,7 +12,7 @@ from osmaxx.conversion import models as conversion_models from osmaxx.conversion._settings import CONVERSION_SETTINGS -from osmaxx.conversion_api.statuses import FINAL_STATUSES, FINISHED, FAILED +from osmaxx.conversion.constants.statuses import FINAL_STATUSES, FINISHED, FAILED logging.basicConfig() logger = logging.getLogger(__name__) diff --git a/osmaxx/conversion/models.py b/osmaxx/conversion/models.py index 6421ad503..3901ae5fa 100644 --- a/osmaxx/conversion/models.py +++ b/osmaxx/conversion/models.py @@ -7,12 +7,12 @@ from django.utils.translation import gettext_lazy as _ from rest_framework.reverse import reverse +from osmaxx.conversion import output_format from osmaxx.clipping_area.models import ClippingArea from osmaxx.conversion.converters.converter import convert -from osmaxx.conversion.converters.detail_levels import DETAIL_LEVEL_CHOICES, DETAIL_LEVEL_ALL -from osmaxx.conversion_api.coordinate_reference_systems import CRS_CHOICES -from osmaxx.conversion_api.formats import FORMAT_CHOICES -from osmaxx.conversion_api.statuses import STATUS_CHOICES, RECEIVED +from osmaxx.conversion.converters.converter_gis.detail_levels import DETAIL_LEVEL_CHOICES, DETAIL_LEVEL_ALL +from osmaxx.conversion.constants.coordinate_reference_systems import CRS_CHOICES +from osmaxx.conversion.constants.statuses import STATUS_CHOICES, RECEIVED def job_directory_path(instance, filename): @@ -20,7 +20,7 @@ def job_directory_path(instance, filename): class Parametrization(models.Model): - out_format = models.CharField(verbose_name=_("out format"), choices=FORMAT_CHOICES, max_length=100) + out_format = models.CharField(verbose_name=_("out format"), choices=output_format.CHOICES, max_length=100) out_srs = models.IntegerField( verbose_name=_("output SRS"), help_text=_("EPSG code of the output spatial reference system"), null=True, blank=True, default=4326, choices=CRS_CHOICES diff --git a/osmaxx/conversion/serializers.py b/osmaxx/conversion/serializers.py index 4bfef9511..1d308d41a 100644 --- a/osmaxx/conversion/serializers.py +++ b/osmaxx/conversion/serializers.py @@ -1,8 +1,8 @@ from rest_framework import serializers -from osmaxx.conversion.converters import detail_levels +from osmaxx.conversion import output_format +from osmaxx.conversion.converters.converter_gis import detail_levels from osmaxx.conversion.size_estimator import size_estimation_for_format -from osmaxx.conversion_api import formats from .models import Job, Parametrization @@ -41,7 +41,7 @@ def validate(self, data): data.update( { output_format: size_estimation_for_format(output_format, detail_level, estimated_pbf) - for output_format in formats.FORMAT_DEFINITIONS + for output_format in output_format.DEFINITIONS } ) return data diff --git a/osmaxx/conversion/size_estimator.py b/osmaxx/conversion/size_estimator.py index 57c9cbeff..69d5aa7a3 100644 --- a/osmaxx/conversion/size_estimator.py +++ b/osmaxx/conversion/size_estimator.py @@ -1,37 +1,36 @@ import math +from osmaxx.conversion import output_format +from osmaxx.conversion.converters.converter_gis import detail_levels from osmaxx.conversion.models import Job -from osmaxx.conversion_api import formats -from osmaxx.conversion.converters import detail_levels - PRE_DATA = { - formats.GARMIN: { + output_format.GARMIN: { 'pbf_predicted': [25000, 44000, 96000, 390000], detail_levels.DETAIL_LEVEL_ALL: [11000, 18000, 42000, 95000], detail_levels.DETAIL_LEVEL_REDUCED: [11000, 18000, 42000, 95000], }, - formats.PBF: { + output_format.PBF: { 'pbf_predicted': [25000, 44000, 96000, 390000], detail_levels.DETAIL_LEVEL_ALL: [25000, 44000, 96000, 390000], detail_levels.DETAIL_LEVEL_REDUCED: [25000, 44000, 96000, 390000], }, - formats.FGDB: { + output_format.FGDB: { 'pbf_predicted': [25000, 44000, 96000, 390000], detail_levels.DETAIL_LEVEL_ALL: [46000, 101000, 309000, 676000], detail_levels.DETAIL_LEVEL_REDUCED: [21000, 27000, 107000, 250000], }, - formats.GPKG: { + output_format.GPKG: { 'pbf_predicted': [25000, 44000, 96000, 390000], detail_levels.DETAIL_LEVEL_ALL: [109000, 210000, 690000, 1500000], detail_levels.DETAIL_LEVEL_REDUCED: [49000, 58000, 252000, 599000], }, - formats.SHAPEFILE: { + output_format.SHAPEFILE: { 'pbf_predicted': [25000, 44000, 96000, 390000], detail_levels.DETAIL_LEVEL_ALL: [255000, 638000, 2000000, 4400000], detail_levels.DETAIL_LEVEL_REDUCED: [100000, 138000, 652000, 1600000], }, - formats.SPATIALITE: { + output_format.SPATIALITE: { 'pbf_predicted': [25000, 44000, 96000, 390000], detail_levels.DETAIL_LEVEL_ALL: [115000, 216000, 719000, 1600000], detail_levels.DETAIL_LEVEL_REDUCED: [55000, 66000, 269000, 635000], @@ -50,7 +49,7 @@ def size_estimation_for_format(format_type, detail_level, predicted_pbf_size): def get_data(format_type, detail_level): - assert format_type in formats.FORMAT_DEFINITIONS + assert format_type in output_format.DEFINITIONS assert detail_level in [level[0] for level in detail_levels.DETAIL_LEVEL_CHOICES] base_query_set = Job.objects.filter( parametrization__out_format=format_type, diff --git a/osmaxx/conversion_api/__init__.py b/osmaxx/conversion_api/__init__.py deleted file mode 100644 index aeff70cec..000000000 --- a/osmaxx/conversion_api/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Interface between OSMaxx frontend (Django-based web client) and the OSMaxx conversion service (mediator and worker). - -This Django app holds shared code. All OSMaxx components (frontend & conversion service) may depend on it. -No code in this app may depend on other OSMaxx Django apps, especially not the component-specific ones. - -The OSMaxx conversion service implementation (provided by the mediator) is located in Django app `conversion`, -not here in app `conversion_api`. -""" diff --git a/tests/utilities/__init__.py b/osmaxx/core/__init__.py similarity index 100% rename from tests/utilities/__init__.py rename to osmaxx/core/__init__.py diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil_wgs-84_2016-11-17_gpkg_full-detail.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_fgdb_full-detail.zip similarity index 71% rename from osmaxx/core/static/osmaxx/example_data/rapperswil_wgs-84_2016-11-17_gpkg_full-detail.zip rename to osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_fgdb_full-detail.zip index 894df7c7d..d5dcfebe4 100644 Binary files a/osmaxx/core/static/osmaxx/example_data/rapperswil_wgs-84_2016-11-17_gpkg_full-detail.zip and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_fgdb_full-detail.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_fgdb_simplified.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_fgdb_simplified.zip new file mode 100644 index 000000000..76056bd99 Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_fgdb_simplified.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_garmin_full-detail.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_garmin_full-detail.zip new file mode 100644 index 000000000..1e208b182 Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_garmin_full-detail.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_gpkg_full-detail.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_gpkg_full-detail.zip new file mode 100644 index 000000000..61b2a4dc8 Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_gpkg_full-detail.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_gpkg_simplified.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_gpkg_simplified.zip new file mode 100644 index 000000000..6bee9cd50 Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_gpkg_simplified.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_pbf_full-detail.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_pbf_full-detail.zip new file mode 100644 index 000000000..c2caf700d Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_pbf_full-detail.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_shapefile_full-detail.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_shapefile_full-detail.zip new file mode 100644 index 000000000..8947060c2 Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_shapefile_full-detail.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_shapefile_simplified.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_shapefile_simplified.zip new file mode 100644 index 000000000..b787250f5 Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_shapefile_simplified.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_spatialite_full-detail.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_spatialite_full-detail.zip new file mode 100644 index 000000000..87b6221ad Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_spatialite_full-detail.zip differ diff --git a/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_spatialite_simplified.zip b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_spatialite_simplified.zip new file mode 100644 index 000000000..a176a1581 Binary files /dev/null and b/osmaxx/core/static/osmaxx/example_data/rapperswil-jona_wgs-84_2017-03-20_spatialite_simplified.zip differ diff --git a/osmaxx/core/static/osmaxx/images/homepage_screenshot.png b/osmaxx/core/static/osmaxx/images/homepage_screenshot.png new file mode 100644 index 000000000..27b4cbf2b Binary files /dev/null and b/osmaxx/core/static/osmaxx/images/homepage_screenshot.png differ diff --git a/osmaxx/core/static/osmaxx/stylesheets/main.css b/osmaxx/core/static/osmaxx/stylesheets/main.css index 37728f0b2..06cff7c95 100644 --- a/osmaxx/core/static/osmaxx/stylesheets/main.css +++ b/osmaxx/core/static/osmaxx/stylesheets/main.css @@ -282,4 +282,4 @@ footer a:link, footer a:visited, footer a:hover, footer a:focus, -footer a:active { color: white; background-color: rgb(50, 110, 160); padding: 4px 4px; } +footer a:active { color: white; } diff --git a/osmaxx/core/templates/base.html b/osmaxx/core/templates/base.html index db8b88860..4f6d19df0 100644 --- a/osmaxx/core/templates/base.html +++ b/osmaxx/core/templates/base.html @@ -1,7 +1,7 @@ {% load staticfiles i18n %} - +{# needed for Open Graph metatags in osmaxx/core/templates/osmaxx/base.html #} diff --git a/osmaxx/core/templates/osmaxx/base.html b/osmaxx/core/templates/osmaxx/base.html index 84fff4374..a0251ade7 100644 --- a/osmaxx/core/templates/osmaxx/base.html +++ b/osmaxx/core/templates/osmaxx/base.html @@ -1,9 +1,20 @@ {% extends 'base.html' %} -{% load staticfiles %} +{% load staticfiles navigation %} {% block extra_head %} OSMaxx + + + + + + + + + + + diff --git a/osmaxx/core/templates/osmaxx/login.html b/osmaxx/core/templates/osmaxx/login.html index 85aebde14..083e19d91 100644 --- a/osmaxx/core/templates/osmaxx/login.html +++ b/osmaxx/core/templates/osmaxx/login.html @@ -22,6 +22,6 @@

{% trans 'Login' %}

{% endblocktrans %}

- OSM Logo Log in using OpenStreetMap + OSM Logo Log in using OpenStreetMap {% endblock %} diff --git a/osmaxx/core/templates/pages/downloads.html b/osmaxx/core/templates/pages/downloads.html index 8a33879b9..0882afbc3 100644 --- a/osmaxx/core/templates/pages/downloads.html +++ b/osmaxx/core/templates/pages/downloads.html @@ -10,12 +10,60 @@

Downloads

Here you can find additional downloads and some useful links to similar projects.

-

Styles

-

Sample Export

+

Sample Exports

- As an export example, we provide an example export featuring "Rapperswil"
- rapperswil_wgs-84_2016-11-17_gpkg_full-detail.zip. + The following sample exports feature excerpt "Rapperswil-Jona". +

Full detail GIS format exports

+
+
Esri File Geodatabase
+
+ rapperswil-jona_wgs-84_2017-03-20_fgdb_full-detail.zip +
+
Esri Shapefile
+
+ rapperswil-jona_wgs-84_2017-03-20_shapefile_full-detail.zip +
+
GeoPackage
+
+ rapperswil-jona_wgs-84_2017-03-20_gpkg_full-detail.zip +
+
SpatiaLite
+
+ rapperswil-jona_wgs-84_2017-03-20_spatialite_full-detail.zip +
+
+

Reduced detail ("simplified") GIS format exports

+
+
Esri File Geodatabase
+
+ rapperswil-jona_wgs-84_2017-03-20_fgdb_simplified.zip +
+
Esri Shapefile
+
+ rapperswil-jona_wgs-84_2017-03-20_shapefile_simplified.zip +
+
GeoPackage
+
+ rapperswil-jona_wgs-84_2017-03-20_gpkg_simplified.zip +
+
SpatiaLite
+
+ rapperswil-jona_wgs-84_2017-03-20_spatialite_simplified.zip +
+
+

Non-GIS formats

+
+
Garmin navigation & map data
+
+ rapperswil-jona_wgs-84_2017-03-20_garmin_full-detail.zip +
+
OSM Protocolbuffer Binary Format
+
+ rapperswil-jona_wgs-84_2017-03-20_pbf_full-detail.zip +
+

+

Styles

QGIS

A QGIS project file is being delivered with every export, matching the exported data containing the OMSaxx diff --git a/tests/utilities/test_models/__init__.py b/osmaxx/core/templatetags/__init__.py similarity index 100% rename from tests/utilities/test_models/__init__.py rename to osmaxx/core/templatetags/__init__.py diff --git a/osmaxx/core/templatetags/navigation.py b/osmaxx/core/templatetags/navigation.py new file mode 100644 index 000000000..211bafe7c --- /dev/null +++ b/osmaxx/core/templatetags/navigation.py @@ -0,0 +1,34 @@ +from urllib.parse import urlsplit, urlunsplit + +import logging +from django import template +from django.conf import settings +from django.http import HttpRequest + +logger = logging.getLogger(__name__) + +register = template.Library() + + +@register.filter +def siteabsoluteurl(relative_or_absolute_url: str, request: HttpRequest) -> str: + """ + Produce URL suitable for offsite use. + + Args: + relative_or_absolute_url: A relative, scheme-relative or absolute URL + request: A Django request object, from which to take the missing parts + + Returns: + An absolute URL, incl. scheme, host, etc. + """ + absolute_url = request.build_absolute_uri(relative_or_absolute_url) + if settings.OSMAXX.get('SECURED_PROXY', False): + url_components = urlsplit(absolute_url) + if url_components.scheme.startswith('http'): + absolute_url = urlunsplit(url_components._replace(scheme='https')) + elif logger.getEffectiveLevel() == logging.WARNING: + logger.warning( + "{url} has not been converted to HTTPS, because it isn't an HTTP URL.".format(url=absolute_url) + ) + return absolute_url diff --git a/osmaxx/excerptexport/forms/excerpt_form.py b/osmaxx/excerptexport/forms/excerpt_form.py index 49329c2f8..216f16223 100644 --- a/osmaxx/excerptexport/forms/excerpt_form.py +++ b/osmaxx/excerptexport/forms/excerpt_form.py @@ -1,16 +1,15 @@ import json -from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ -from django import forms -from django.core.urlresolvers import reverse - from crispy_forms.helper import FormHelper from crispy_forms.layout import Layout, Fieldset, Field, Submit +from django import forms +from django.core.exceptions import ValidationError +from django.core.urlresolvers import reverse +from django.utils.translation import ugettext_lazy as _ -from .order_options_mixin import OrderOptionsMixin from osmaxx.excerptexport.models import Excerpt, ExtractionOrder -from osmaxx.utilities.dict_helpers import select_keys +from osmaxx.utils.dict_helpers import select_keys +from .order_options_mixin import OrderOptionsMixin class ExcerptForm(OrderOptionsMixin, forms.ModelForm): @@ -26,7 +25,7 @@ class ExcerptForm(OrderOptionsMixin, forms.ModelForm): ) def __init__(self, *args, **kwargs): - super(ExcerptForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.helper = FormHelper(self) @@ -41,7 +40,7 @@ def __init__(self, *args, **kwargs): Field('bounding_geometry'), ), OrderOptionsMixin(self).form_layout(), - Submit('submit', 'Submit'), + Submit('submit', 'Export (will take around 30 minutes)'), ) def clean(self): diff --git a/osmaxx/excerptexport/forms/existing_form.py b/osmaxx/excerptexport/forms/existing_form.py index 185f795e6..6bc16f8cf 100644 --- a/osmaxx/excerptexport/forms/existing_form.py +++ b/osmaxx/excerptexport/forms/existing_form.py @@ -28,7 +28,7 @@ def _choicify(excerpts_query_set): class ExistingForm(OrderOptionsMixin, forms.Form): def __init__(self, *args, **kwargs): - super(ExistingForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.helper = FormHelper(self) @@ -44,7 +44,7 @@ def __init__(self, *args, **kwargs): HTML(render_to_string('excerptexport/forms/partials/delete_personal_excerpts_link.html')) ), OrderOptionsMixin(self).form_layout(), - Submit('submit', 'Submit'), + Submit('submit', 'Export (will take around 30 minutes)'), ) @classmethod diff --git a/osmaxx/excerptexport/forms/order_options_mixin.py b/osmaxx/excerptexport/forms/order_options_mixin.py index 23f27d7b1..86c53627b 100644 --- a/osmaxx/excerptexport/forms/order_options_mixin.py +++ b/osmaxx/excerptexport/forms/order_options_mixin.py @@ -1,16 +1,16 @@ -from django.utils.translation import ugettext_lazy as _ -from django import forms - from crispy_forms.layout import Fieldset, Div +from django import forms +from django.utils.translation import ugettext_lazy as _ -from osmaxx.conversion.converters.detail_levels import DETAIL_LEVEL_CHOICES -from osmaxx.conversion_api import formats, coordinate_reference_systems as crs +from osmaxx.conversion import output_format +from osmaxx.conversion.converters.converter_gis.detail_levels import DETAIL_LEVEL_CHOICES +from osmaxx.conversion.constants import coordinate_reference_systems as crs class OrderOptionsMixin(forms.Form): formats = forms.MultipleChoiceField( label=_("GIS export formats"), - choices=formats.FORMAT_CHOICES, + choices=output_format.CHOICES, widget=forms.CheckboxSelectMultiple, required=True, ) diff --git a/osmaxx/excerptexport/models/export.py b/osmaxx/excerptexport/models/export.py index d929fc403..45e5d6614 100644 --- a/osmaxx/excerptexport/models/export.py +++ b/osmaxx/excerptexport/models/export.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework.reverse import reverse -from osmaxx.conversion_api.formats import FORMAT_CHOICES +from osmaxx.conversion import output_format from osmaxx.excerptexport._settings import RESULT_FILE_AVAILABILITY_DURATION, EXTRACTION_PROCESSING_TIMEOUT_TIMEDELTA logger = logging.getLogger(__name__) @@ -41,14 +41,14 @@ class Export(TimeStampModelMixin, models.Model): - the transformation of the data from the data sources' schemata (e.g. ``osm2pgsql`` schema) to the OSMaxx schema - the actual export to one specific GIS or navigation file format with one specific set of parameters """ - from osmaxx.conversion_api.statuses import RECEIVED, QUEUED, FINISHED, FAILED, STARTED, DEFERRED, FINAL_STATUSES, STATUS_CHOICES # noqa + from osmaxx.conversion.constants.statuses import RECEIVED, QUEUED, FINISHED, FAILED, STARTED, DEFERRED, FINAL_STATUSES, STATUS_CHOICES # noqa INITIAL = 'initial' INITIAL_CHOICE = (INITIAL, _('initial')) STATUS_CHOICES = (INITIAL_CHOICE,) + STATUS_CHOICES extraction_order = models.ForeignKey('excerptexport.ExtractionOrder', related_name='exports', verbose_name=_('extraction order'), on_delete=models.CASCADE) - file_format = models.CharField(choices=FORMAT_CHOICES, verbose_name=_('file format / data format'), max_length=10) + file_format = models.CharField(choices=output_format.CHOICES, verbose_name=_('file format / data format'), max_length=10) conversion_service_job_id = models.IntegerField(verbose_name=_('conversion service job ID'), null=True) status = models.CharField(_('job status'), choices=STATUS_CHOICES, default=INITIAL, max_length=20) finished_at = models.DateTimeField(_('finished at'), default=None, blank=True, editable=False, null=True) @@ -97,7 +97,7 @@ def update_is_overdue(self): return (self.updated_at + EXTRACTION_PROCESSING_TIMEOUT_TIMEDELTA) < timezone.now() def _handle_changed_status(self, *, incoming_request): - from osmaxx.utilities.shortcuts import Emissary + from osmaxx.utils.shortcuts import Emissary emissary = Emissary(recipient=self.extraction_order.orderer) status_changed_message = self._get_export_status_changed_message() if self.status == self.FAILED: diff --git a/osmaxx/excerptexport/models/extraction_order.py b/osmaxx/excerptexport/models/extraction_order.py index e0bdc965c..bc0f4adf8 100644 --- a/osmaxx/excerptexport/models/extraction_order.py +++ b/osmaxx/excerptexport/models/extraction_order.py @@ -3,10 +3,11 @@ from django.db.models.signals import post_save from django.dispatch.dispatcher import receiver from django.template.loader import render_to_string +from django.utils.text import unescape_entities from django.utils.translation import ugettext_lazy as _ -from osmaxx.conversion.converters.detail_levels import DETAIL_LEVEL_CHOICES, DETAIL_LEVEL_ALL -from osmaxx.conversion_api import coordinate_reference_systems as crs +from osmaxx.conversion.converters.converter_gis.detail_levels import DETAIL_LEVEL_CHOICES, DETAIL_LEVEL_ALL +from osmaxx.conversion.constants import coordinate_reference_systems as crs from .excerpt import Excerpt @@ -73,7 +74,7 @@ def get_absolute_url(self): def send_email_if_all_exports_done(self, incoming_request): if all(export.is_status_final for export in self.exports.all()): - from osmaxx.utilities.shortcuts import Emissary + from osmaxx.utils.shortcuts import Emissary emissary = Emissary(recipient=self.orderer) emissary.inform_mail( subject=self._get_all_exports_done_email_subject(), @@ -86,10 +87,12 @@ def _get_all_exports_done_email_subject(self): successful_exports_count=self.exports.filter(output_file__isnull=False).count(), failed_exports_count=self.exports.filter(output_file__isnull=True).count(), ) - return render_to_string( - 'excerptexport/email/all_exports_of_extraction_order_done_subject.txt', - context=view_context, - ).strip() + return unescape_entities( + render_to_string( + 'excerptexport/email/all_exports_of_extraction_order_done_subject.txt', + context=view_context, + ).strip() + ) # HACK: calling unescape_entities as workaround for https://github.com/geometalab/osmaxx/issues/771 def _get_all_exports_done_mail_body(self, incoming_request): view_context = dict( @@ -98,10 +101,12 @@ def _get_all_exports_done_mail_body(self, incoming_request): failed_exports=self.exports.filter(output_file__isnull=True), request=incoming_request, ) - return render_to_string( - 'excerptexport/email/all_exports_of_extraction_order_done_body.txt', - context=view_context, - ).strip() + return unescape_entities( + render_to_string( + 'excerptexport/email/all_exports_of_extraction_order_done_body.txt', + context=view_context, + ).strip() + ) # HACK: calling unescape_entities as workaround for https://github.com/geometalab/osmaxx/issues/771 @receiver(post_save, sender=ExtractionOrder) diff --git a/osmaxx/excerptexport/static/excerptexport/images/define_and_export_new_excerpt_emblem.svg b/osmaxx/excerptexport/static/excerptexport/images/define_and_export_new_excerpt_icon.svg similarity index 100% rename from osmaxx/excerptexport/static/excerptexport/images/define_and_export_new_excerpt_emblem.svg rename to osmaxx/excerptexport/static/excerptexport/images/define_and_export_new_excerpt_icon.svg diff --git a/osmaxx/excerptexport/static/excerptexport/images/export_existing_excerpt_or_country_or_admin_area_emblem.svg b/osmaxx/excerptexport/static/excerptexport/images/export_existing_excerpt_or_country_or_admin_area_icon.svg similarity index 100% rename from osmaxx/excerptexport/static/excerptexport/images/export_existing_excerpt_or_country_or_admin_area_emblem.svg rename to osmaxx/excerptexport/static/excerptexport/images/export_existing_excerpt_or_country_or_admin_area_icon.svg diff --git a/osmaxx/excerptexport/templates/excerptexport/email/all_exports_of_extraction_order_done_body.txt b/osmaxx/excerptexport/templates/excerptexport/email/all_exports_of_extraction_order_done_body.txt index 00f4c8cec..671f244f5 100644 --- a/osmaxx/excerptexport/templates/excerptexport/email/all_exports_of_extraction_order_done_body.txt +++ b/osmaxx/excerptexport/templates/excerptexport/email/all_exports_of_extraction_order_done_body.txt @@ -1,12 +1,14 @@ {% load navigation %}This is an automated email from {{ request.get_host }} -The extraction order #{{ extraction_order.id }} "{{ extraction_order.excerpt_name }}" has been processed{% if successful_exports.count > 0 %} and is available for download: -{% for export in successful_exports %}- {{ export.get_file_format_display }}{% if export.output_file.file %}: {{ export.output_file.file.url | siteabsoluteurl:request }}{% endif %} +The extraction order #{{ extraction_order.id }} "{{ extraction_order.excerpt_name }}" has been processed{% for export in successful_exports %}{% if forloop.first %} and is available for download: +{% endif %}- {{ export.get_file_format_display }}{% if export.output_file.file %}: {{ export.output_file.file.url | siteabsoluteurl:request }}{% endif %} +{% empty %}. {% endfor %} -{% endif %}{% if failed_exports.count > 0 %}Unfortunately, the following exports have failed: -{% for export in failed_exports %}- {{ export.get_file_format_display }} -{% endfor %}Please order them anew if you need them. If there are repeated failures, please report them on https://github.com/geometalab/osmaxx/issues unless the problem is already known there. -{% endif %} +{% for export in failed_exports %}{% if forloop.first %}Unfortunately, the following export{{ failed_exports|pluralize }} ha{{ failed_exports|pluralize:"s,ve" }} failed: +{% endif %}- {{ export.get_file_format_display }} +{% if forloop.last %} +Please order {{ failed_exports|pluralize:"it,them" }} anew if you need {{ failed_exports|pluralize:"it,them" }}. If there are repeated failures, please report them on https://github.com/geometalab/osmaxx/issues unless the problem is already known there. +{% endif %}{% endfor %} View the complete order at {{ extraction_order.get_absolute_url | siteabsoluteurl:request }} (login required) Thank you for using OSMaxx. diff --git a/osmaxx/excerptexport/templates/excerptexport/partials/content_file_type_description.html b/osmaxx/excerptexport/templates/excerptexport/partials/content_file_type_description.html index b46996c27..d9b4fa196 100644 --- a/osmaxx/excerptexport/templates/excerptexport/partials/content_file_type_description.html +++ b/osmaxx/excerptexport/templates/excerptexport/partials/content_file_type_description.html @@ -1,4 +1,4 @@ -{% load filter_filename i18n %} +{% load i18n %} {% spaceless %} {% with content_type|lower as type %} {% if type == 'gpkg' %} diff --git a/osmaxx/excerptexport/templates/excerptexport/partials/export_detail_string.html b/osmaxx/excerptexport/templates/excerptexport/partials/export_detail_string.html index 255eed6ca..278d13406 100644 --- a/osmaxx/excerptexport/templates/excerptexport/partials/export_detail_string.html +++ b/osmaxx/excerptexport/templates/excerptexport/partials/export_detail_string.html @@ -1,7 +1,7 @@ {% load i18n capture_as %}{{ export.get_file_format_display }} / -{% firstof export.finished_at|date:"Y-m-d H:i:s" export.updated_at|date:"Y-m-d H:i:s" as finished_date %} -{% capture_as state_display %}{{ export.status }} at {{ finished_date }}{% end_capture_as %} -{% capture_as available_until %}{{ export.result_file_available_until|date:"Y-m-d H:i:s" }}{% end_capture_as %} +{% firstof export.finished_at|date:'DATETIME_FORMAT' export.updated_at|date:'DATETIME_FORMAT' as changed_at_datetime %} +{% capture_as state_display %}{{ export.status }} at {{ changed_at_datetime }}{% end_capture_as %} +{% capture_as available_until %}{{ export.result_file_available_until|date:'DATETIME_FORMAT' }}{% end_capture_as %} {% if export.status == export.FINISHED and export.result_file_available_until %} {% blocktrans %}available until {{ available_until }}{% endblocktrans %} / {% else %} diff --git a/osmaxx/excerptexport/templates/excerptexport/partials/export_display.html b/osmaxx/excerptexport/templates/excerptexport/partials/export_display.html index 55981908f..70c32a31d 100644 --- a/osmaxx/excerptexport/templates/excerptexport/partials/export_display.html +++ b/osmaxx/excerptexport/templates/excerptexport/partials/export_display.html @@ -13,9 +13,9 @@

- {% capture_as finished_date %}{% firstof export.finished_at|date:"Y-m-d H:i:s" export.updated_at|date:"Y-m-d H:i:s" %}{% end_capture_as %} - {% capture_as state_display %}{{ export.status }} at {{ finished_date }}{% end_capture_as %} - {% capture_as available_until %}{{ export.result_file_available_until|date:"Y-m-d H:i:s" }}{% end_capture_as %} + {% capture_as changed_at_datetime %}{% firstof export.finished_at|date:'DATETIME_FORMAT' export.updated_at|date:'DATETIME_FORMAT' %}{% end_capture_as %} + {% capture_as state_display %}{{ export.status }} at {{ changed_at_datetime }}{% end_capture_as %} + {% capture_as available_until %}{{ export.result_file_available_until|date:'DATETIME_FORMAT' }}{% end_capture_as %} {% if export.status == export.FINISHED and export.result_file_available_until %} {% blocktrans %}available until {{ available_until }}{% endblocktrans %} {% else %} diff --git a/osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html b/osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html deleted file mode 100644 index 09b47bbe8..000000000 --- a/osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html +++ /dev/null @@ -1,29 +0,0 @@ -{% extends "excerptexport/base.html" %} - -{% load i18n %} - -{% block main %} -
- {% if user.is_authenticated %} -

{% trans 'Please Contact Your Administrator or Request Access' %}

-

- {% blocktrans %}You do not appear to have the proper access rights. Please contact your - supervisor/administrator and ask for the needed access rights.{% endblocktrans %} -

- -

New account

-

- {% blocktrans %}If you logged in by a new OpenStreetMap-account, please request its activation: {% endblocktrans %}
-

- {% csrf_token %} - -
- {% blocktrans %}and try again.{% endblocktrans %} -

- {% else %} -

{% trans 'Please Log In' %}

-

{% trans 'This site has pages, which are protected. Please login and try again.' %}

- {% trans 'Login' %} - {% endif %} -
-{% endblock %} diff --git a/osmaxx/excerptexport/templates/excerptexport/templates/index.html b/osmaxx/excerptexport/templates/excerptexport/templates/index.html index d18dce7b7..0ead0672f 100644 --- a/osmaxx/excerptexport/templates/excerptexport/templates/index.html +++ b/osmaxx/excerptexport/templates/excerptexport/templates/index.html @@ -18,17 +18,17 @@

-
- - {% trans 'Export an existing excerpt' %} +
+ + {% trans 'Existing Excerpt / Country' %} -
- {% trans '(a predefined country or other administrative area,
or one of your previously defined excerpts)' %}
-
- - {% trans 'Define & export a new excerpt' %} +
+ + {% trans 'New Excerpt' %}
{% if user.is_authenticated %} @@ -45,17 +45,14 @@

{% block copyright %} {{ block.super }}, - - "folder with map" - + + "folder with map" and - - "cut-out from planet" emblems - + + "cut-out from planet" icons derived from CC BY-SA and public domain works and licensed under - CC BY-SA 4.0 international - + CC BY-SA 4.0 international {% endblock %} diff --git a/osmaxx/excerptexport/templatetags/filter_filename.py b/osmaxx/excerptexport/templatetags/filter_filename.py deleted file mode 100644 index 80963b277..000000000 --- a/osmaxx/excerptexport/templatetags/filter_filename.py +++ /dev/null @@ -1,11 +0,0 @@ -import os - -from django import template - - -register = template.Library() - - -@register.filter -def filter_filename(file_object): - return os.path.basename(file_object.name) if file_object and file_object.name else file_object diff --git a/osmaxx/excerptexport/templatetags/navigation.py b/osmaxx/excerptexport/templatetags/navigation.py deleted file mode 100644 index c0fac22e5..000000000 --- a/osmaxx/excerptexport/templatetags/navigation.py +++ /dev/null @@ -1,19 +0,0 @@ -# Adopted from https://djangosnippets.org/snippets/2783/ -from django import template -from django.conf import settings - -register = template.Library() - - -@register.filter -def siteabsoluteurl(absoluteurl, request): - """ - receives an argument in the form of an absolute url - Returns the same url with all the necessary path information to be used offsite - """ - absolute_url = request.build_absolute_uri(absoluteurl) - if settings.OSMAXX.get('SECURED_PROXY', False) and not absolute_url.startswith('https'): - if absolute_url.startswith('http'): - absolute_url = absolute_url[4:] - absolute_url = 'https' + absolute_url - return absolute_url diff --git a/osmaxx/excerptexport/urls.py b/osmaxx/excerptexport/urls.py index 685831cd9..ee69d4394 100644 --- a/osmaxx/excerptexport/urls.py +++ b/osmaxx/excerptexport/urls.py @@ -3,20 +3,17 @@ from django.views.generic import TemplateView from osmaxx.excerptexport.views import ( - access_denied, delete_excerpt, export_list, export_detail, manage_own_excerpts, order_new_excerpt, order_existing_excerpt, - request_access, ) excerpt_export_urlpatterns = [ url(r'^$', TemplateView.as_view(template_name="excerptexport/templates/index.html"), name='index'), - url(r'^access_denied/$', access_denied, name='access_denied'), url(r'^exports/$', export_list, name='export_list'), url(r'^exports/(?P[A-Za-z0-9_-]+)/$', export_detail, name='export_detail'), @@ -25,8 +22,6 @@ url(r'^excerpts/(?P[A-Za-z0-9_-]+)/delete/$', delete_excerpt, name='delete_excerpt'), url(r'^excerpts/$', manage_own_excerpts, name='manage_own_excerpts'), - - url(r'^request_access/$', request_access, name='request_access'), ] login_logout_patterns = [ diff --git a/osmaxx/excerptexport/views.py b/osmaxx/excerptexport/views.py index 1ae38564e..e9e219d47 100644 --- a/osmaxx/excerptexport/views.py +++ b/osmaxx/excerptexport/views.py @@ -1,27 +1,20 @@ import logging from collections import OrderedDict -from django.conf import settings from django.contrib import messages -from django.contrib.auth.decorators import login_required +from django.contrib.auth.mixins import LoginRequiredMixin from django.core.exceptions import PermissionDenied -from django.core.mail import send_mail from django.core.urlresolvers import reverse from django.http import HttpResponseRedirect -from django.shortcuts import redirect from django.utils.datastructures import OrderedSet from django.utils.translation import ugettext_lazy as _ -from django.views.generic import FormView, TemplateView, GenericViewError +from django.views.generic import FormView, GenericViewError from django.views.generic.detail import SingleObjectMixin from django.views.generic.edit import FormMixin, DeleteView from django.views.generic.list import ListView -from osmaxx.contrib.auth.frontend_permissions import ( - LoginRequiredMixin, - FrontendAccessRequiredMixin, - EmailRequiredMixin, -) -from osmaxx.conversion_api import statuses +from osmaxx.contrib.auth.frontend_permissions import EmailRequiredMixin +from osmaxx.conversion.constants import statuses from osmaxx.excerptexport.forms import ExcerptForm, ExistingForm from osmaxx.excerptexport.models import Excerpt from osmaxx.excerptexport.models import ExtractionOrder @@ -51,17 +44,13 @@ def form_valid(self, form): ) -class AccessRestrictedBaseView(LoginRequiredMixin, FrontendAccessRequiredMixin, EmailRequiredMixin): - pass - - -class OrderNewExcerptView(AccessRestrictedBaseView, OrderFormViewMixin, FormView): +class OrderNewExcerptView(LoginRequiredMixin, EmailRequiredMixin, OrderFormViewMixin, FormView): template_name = 'excerptexport/templates/order_new_excerpt.html' form_class = ExcerptForm order_new_excerpt = OrderNewExcerptView.as_view() -class OrderExistingExcerptView(AccessRestrictedBaseView, OrderFormViewMixin, FormView): +class OrderExistingExcerptView(LoginRequiredMixin, EmailRequiredMixin, OrderFormViewMixin, FormView): template_name = 'excerptexport/templates/order_existing_excerpt.html' form_class = ExistingForm @@ -117,7 +106,7 @@ def _get_extra_context_data(self): ) -class ExportsListView(AccessRestrictedBaseView, ExportsListMixin, ListView): +class ExportsListView(LoginRequiredMixin, ExportsListMixin, ListView): template_name = 'excerptexport/export_list.html' context_object_name = 'excerpts' model = Excerpt @@ -145,7 +134,7 @@ def _get_exports_for_excerpt(self, excerpt): export_list = ExportsListView.as_view() -class ExportsDetailView(AccessRestrictedBaseView, ExportsListMixin, ListView): +class ExportsDetailView(LoginRequiredMixin, ExportsListMixin, ListView): template_name = 'excerptexport/export_detail.html' context_object_name = 'exports' model = Export @@ -168,61 +157,6 @@ def get_queryset(self): export_detail = ExportsDetailView.as_view() -class AccesssDenied(TemplateView): - template_name = 'excerptexport/templates/access_denied.html' - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context.update(dict(next_page=self.request.GET.get('next', '/'))) -access_denied = AccesssDenied.as_view() - - -@login_required() -def request_access(request): - user_administrator_email = settings.OSMAXX['ACCOUNT_MANAGER_EMAIL'] - if not user_administrator_email: - logging.exception( - "You don't have an user account manager email address defined. Please set OSMAXX_ACCOUNT_MANAGER_EMAIL." - ) - messages.error( - request, - _('Sending of access request failed. Please contact an administrator.') - ) - else: - email_message = ( # Intentionally untranslated, as this goes to the administrator(s), not the user. - '''Hi Admin! - User '{username}' ({identification_description}) claims to be {first_name} {last_name} ({email}) - and requests access for OSMaxx. - If {username} shall be granted access, go to {admin_url} and add {username} to group '{frontend_group}'. - ''' - ).format( - username=request.user.username, - first_name=request.user.first_name, - last_name=request.user.last_name, - email=request.user.email, - identification_description=_social_identification_description(request.user), - admin_url=request.build_absolute_uri(reverse('admin:auth_user_change', args=(request.user.id,))), - frontend_group=settings.OSMAXX_FRONTEND_USER_GROUP, - ) - - try: - send_mail('Request access for OSMaxx', email_message, settings.DEFAULT_FROM_EMAIL, - [user_administrator_email], fail_silently=True) - messages.success( - request, - _('Your access request has been sent successfully. ' - 'You will receive an email when your account is ready.') - ) - except Exception as exception: - logging.exception("Sending access request e-mail failed: {0}, \n{1}".format(exception, email_message)) - messages.error( - request, - _('Sending of access request failed. Please contact an administrator.') - ) - - return redirect(request.GET['next'] + '?next=' + request.GET['next']) - - def _social_identification_description(user): social_identities = list(user.social_auth.all()) if social_identities: @@ -273,5 +207,5 @@ def delete(self, request, *args, **kwargs): ' Please try deleting again, when these are finished.')) return HttpResponseRedirect(self.get_success_url()) - return super(DeleteExcerptView, self).delete(request, *args, **kwargs) + return super().delete(request, *args, **kwargs) delete_excerpt = DeleteExcerptView.as_view() diff --git a/osmaxx/job_progress/middleware.py b/osmaxx/job_progress/middleware.py index 3c08200bb..b527ba3bb 100644 --- a/osmaxx/job_progress/middleware.py +++ b/osmaxx/job_progress/middleware.py @@ -5,9 +5,9 @@ from requests import HTTPError from osmaxx.api_client import ConversionApiClient -from osmaxx.conversion_api.statuses import FINAL_STATUSES, FAILED +from osmaxx.conversion.constants.statuses import FINAL_STATUSES, FAILED from osmaxx.excerptexport.models import Export -from osmaxx.utilities.shortcuts import get_cached_or_set +from osmaxx.utils.shortcuts import get_cached_or_set logger = logging.getLogger(__name__) diff --git a/osmaxx/job_progress/templates/job_progress/messages/export_status_changed.unsave_text b/osmaxx/job_progress/templates/job_progress/messages/export_status_changed.unsave_text index df608006a..852fd4643 100644 --- a/osmaxx/job_progress/templates/job_progress/messages/export_status_changed.unsave_text +++ b/osmaxx/job_progress/templates/job_progress/messages/export_status_changed.unsave_text @@ -1,3 +1,3 @@ {% autoescape off %} -Export #{{ export.id }} "{{ export.extraction_order.excerpt_name }}" to {{ export.get_file_format_display }} has {% if not export.is_status_final %}been {% endif %}{{ export.status }}. +Export #{{ export.id }} "{{ export.extraction_order.excerpt_name }}" to {{ export.get_file_format_display }} has {% if not export.is_status_final %}been {% endif %}{{ export.status }}.{% if export.status == export.STARTED %} Exporting will take around 30 minutes.{% endif %} {% endautoescape %} diff --git a/osmaxx/locale/en/LC_MESSAGES/django.mo b/osmaxx/locale/en/LC_MESSAGES/django.mo index 4d0943b62..37907f3dc 100644 Binary files a/osmaxx/locale/en/LC_MESSAGES/django.mo and b/osmaxx/locale/en/LC_MESSAGES/django.mo differ diff --git a/osmaxx/locale/en/LC_MESSAGES/django.po b/osmaxx/locale/en/LC_MESSAGES/django.po index dc73cc063..575b73e66 100644 --- a/osmaxx/locale/en/LC_MESSAGES/django.po +++ b/osmaxx/locale/en/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-02-02 09:37+0100\n" +"POT-Creation-Date: 2017-05-04 13:59+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -40,11 +40,11 @@ msgid "" "Invalid coordinates: expected at least one coordinate pair, received none." msgstr "" -#: osmaxx/conversion/converters/detail_levels.py:9 +#: osmaxx/conversion/converters/converter_gis/detail_levels.py:9 msgid "Full detail" msgstr "" -#: osmaxx/conversion/converters/detail_levels.py:10 +#: osmaxx/conversion/converters/converter_gis/detail_levels.py:10 msgid "Simplified" msgstr "" @@ -65,7 +65,7 @@ msgid "Clipping Area" msgstr "" #: osmaxx/conversion/models.py:29 -#: osmaxx/excerptexport/models/extraction_order.py:18 +#: osmaxx/excerptexport/models/extraction_order.py:19 msgid "detail level" msgstr "" @@ -133,559 +133,563 @@ msgstr "" msgid "job {} with rq_id {} ({})" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:12 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:12 msgid "WGS 84" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:13 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:13 msgid "Pseudo-Mercator" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:14 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:14 msgid "WGS 72" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:15 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:15 msgid "NAD 83" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:16 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:16 msgid "OSGB 36" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:20 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:20 msgid "UTM Zone 1, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:21 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:21 msgid "UTM Zone 2, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:22 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:22 msgid "UTM Zone 3, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:23 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:23 msgid "UTM Zone 4, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:24 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:24 msgid "UTM Zone 5, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:25 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:25 msgid "UTM Zone 6, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:26 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:26 msgid "UTM Zone 7, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:27 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:27 msgid "UTM Zone 8, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:28 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:28 msgid "UTM Zone 9, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:29 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:29 msgid "UTM Zone 10, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:31 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:31 msgid "UTM Zone 11, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:32 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:32 msgid "UTM Zone 12, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:33 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:33 msgid "UTM Zone 13, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:34 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:34 msgid "UTM Zone 14, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:35 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:35 msgid "UTM Zone 15, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:36 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:36 msgid "UTM Zone 16, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:37 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:37 msgid "UTM Zone 17, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:38 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:38 msgid "UTM Zone 18, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:39 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:39 msgid "UTM Zone 19, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:40 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:40 msgid "UTM Zone 20, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:42 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:42 msgid "UTM Zone 21, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:43 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:43 msgid "UTM Zone 22, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:44 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:44 msgid "UTM Zone 23, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:45 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:45 msgid "UTM Zone 24, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:46 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:46 msgid "UTM Zone 25, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:47 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:47 msgid "UTM Zone 26, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:48 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:48 msgid "UTM Zone 27, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:49 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:49 msgid "UTM Zone 28, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:50 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:50 msgid "UTM Zone 29, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:51 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:51 msgid "UTM Zone 30, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:53 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:53 msgid "UTM Zone 31, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:54 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:54 msgid "UTM Zone 32, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:55 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:55 msgid "UTM Zone 33, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:56 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:56 msgid "UTM Zone 34, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:57 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:57 msgid "UTM Zone 35, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:58 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:58 msgid "UTM Zone 36, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:59 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:59 msgid "UTM Zone 37, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:60 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:60 msgid "UTM Zone 38, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:61 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:61 msgid "UTM Zone 39, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:62 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:62 msgid "UTM Zone 40, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:64 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:64 msgid "UTM Zone 41, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:65 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:65 msgid "UTM Zone 42, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:66 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:66 msgid "UTM Zone 43, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:67 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:67 msgid "UTM Zone 44, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:68 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:68 msgid "UTM Zone 45, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:69 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:69 msgid "UTM Zone 46, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:70 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:70 msgid "UTM Zone 47, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:71 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:71 msgid "UTM Zone 48, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:72 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:72 msgid "UTM Zone 49, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:73 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:73 msgid "UTM Zone 50, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:75 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:75 msgid "UTM Zone 51, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:76 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:76 msgid "UTM Zone 52, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:77 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:77 msgid "UTM Zone 53, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:78 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:78 msgid "UTM Zone 54, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:79 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:79 msgid "UTM Zone 55, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:80 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:80 msgid "UTM Zone 56, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:81 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:81 msgid "UTM Zone 57, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:82 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:82 msgid "UTM Zone 58, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:83 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:83 msgid "UTM Zone 59, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:84 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:84 msgid "UTM Zone 60, northern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:86 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:86 msgid "UTM Zone 1, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:87 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:87 msgid "UTM Zone 2, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:88 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:88 msgid "UTM Zone 3, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:89 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:89 msgid "UTM Zone 4, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:90 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:90 msgid "UTM Zone 5, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:91 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:91 msgid "UTM Zone 6, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:92 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:92 msgid "UTM Zone 7, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:93 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:93 msgid "UTM Zone 8, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:94 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:94 msgid "UTM Zone 9, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:95 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:95 msgid "UTM Zone 10, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:97 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:97 msgid "UTM Zone 11, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:98 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:98 msgid "UTM Zone 12, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:99 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:99 msgid "UTM Zone 13, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:100 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:100 msgid "UTM Zone 14, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:101 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:101 msgid "UTM Zone 15, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:102 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:102 msgid "UTM Zone 16, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:103 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:103 msgid "UTM Zone 17, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:104 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:104 msgid "UTM Zone 18, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:105 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:105 msgid "UTM Zone 19, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:106 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:106 msgid "UTM Zone 20, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:108 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:108 msgid "UTM Zone 21, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:109 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:109 msgid "UTM Zone 22, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:110 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:110 msgid "UTM Zone 23, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:111 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:111 msgid "UTM Zone 24, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:112 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:112 msgid "UTM Zone 25, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:113 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:113 msgid "UTM Zone 26, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:114 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:114 msgid "UTM Zone 27, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:115 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:115 msgid "UTM Zone 28, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:116 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:116 msgid "UTM Zone 29, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:117 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:117 msgid "UTM Zone 30, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:119 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:119 msgid "UTM Zone 31, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:120 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:120 msgid "UTM Zone 32, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:121 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:121 msgid "UTM Zone 33, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:122 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:122 msgid "UTM Zone 34, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:123 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:123 msgid "UTM Zone 35, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:124 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:124 msgid "UTM Zone 36, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:125 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:125 msgid "UTM Zone 37, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:126 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:126 msgid "UTM Zone 38, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:127 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:127 msgid "UTM Zone 39, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:128 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:128 msgid "UTM Zone 40, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:130 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:130 msgid "UTM Zone 41, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:131 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:131 msgid "UTM Zone 42, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:132 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:132 msgid "UTM Zone 43, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:133 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:133 msgid "UTM Zone 44, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:134 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:134 msgid "UTM Zone 45, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:135 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:135 msgid "UTM Zone 46, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:136 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:136 msgid "UTM Zone 47, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:137 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:137 msgid "UTM Zone 48, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:138 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:138 msgid "UTM Zone 49, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:139 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:139 msgid "UTM Zone 50, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:141 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:141 msgid "UTM Zone 51, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:142 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:142 msgid "UTM Zone 52, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:143 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:143 msgid "UTM Zone 53, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:144 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:144 msgid "UTM Zone 54, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:145 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:145 msgid "UTM Zone 55, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:146 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:146 msgid "UTM Zone 56, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:147 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:147 msgid "UTM Zone 57, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:148 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:148 msgid "UTM Zone 58, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:149 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:149 msgid "UTM Zone 59, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:150 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:150 msgid "UTM Zone 60, southern hemisphere" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:154 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:154 msgid "Global coordinate reference systems" msgstr "" -#: osmaxx/conversion_api/coordinate_reference_systems.py:155 +#: osmaxx/conversion.constants/coordinate_reference_systems.py:155 msgid "UTM zones" msgstr "" -#: osmaxx/conversion_api/formats.py:62 +#: osmaxx/conversion.constants/formats.py:45 #: osmaxx/excerptexport/templates/excerptexport/partials/content_file_type_description.html:11 msgid "Esri File Geodatabase" msgstr "" -#: osmaxx/conversion_api/formats.py:69 +#: osmaxx/conversion.constants/formats.py:52 #: osmaxx/excerptexport/templates/excerptexport/partials/content_file_type_description.html:9 msgid "Esri Shapefile" msgstr "" -#: osmaxx/conversion_api/formats.py:77 +#: osmaxx/conversion.constants/formats.py:60 #: osmaxx/excerptexport/templates/excerptexport/partials/content_file_type_description.html:5 msgid "GeoPackage" msgstr "" -#: osmaxx/conversion_api/formats.py:84 +#: osmaxx/conversion.constants/formats.py:67 msgid "SpatiaLite" msgstr "" -#: osmaxx/conversion_api/formats.py:91 +#: osmaxx/conversion.constants/formats.py:74 #: osmaxx/excerptexport/templates/excerptexport/partials/content_file_type_description.html:13 msgid "Garmin navigation & map data" msgstr "" -#: osmaxx/conversion_api/statuses.py:12 +#: osmaxx/conversion.constants/formats.py:81 +msgid "OSM Protocolbuffer Binary Format" +msgstr "" + +#: osmaxx/conversion.constants/statuses.py:12 msgid "received" msgstr "" -#: osmaxx/conversion_api/statuses.py:14 +#: osmaxx/conversion.constants/statuses.py:14 msgid "queued" msgstr "" -#: osmaxx/conversion_api/statuses.py:15 +#: osmaxx/conversion.constants/statuses.py:15 msgid "finished" msgstr "" -#: osmaxx/conversion_api/statuses.py:16 +#: osmaxx/conversion.constants/statuses.py:16 msgid "failed" msgstr "" -#: osmaxx/conversion_api/statuses.py:17 +#: osmaxx/conversion.constants/statuses.py:17 msgid "started" msgstr "" -#: osmaxx/conversion_api/statuses.py:18 +#: osmaxx/conversion.constants/statuses.py:18 msgid "deferred" msgstr "" @@ -693,7 +697,6 @@ msgstr "" #: osmaxx/core/templates/osmaxx/login.html:12 #: osmaxx/core/templates/osmaxx/login_logout_link.html:11 #: osmaxx/core/templates/osmaxx/logout.html:17 -#: osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html:26 msgid "Login" msgstr "" @@ -731,10 +734,12 @@ msgid "Home" msgstr "" #: osmaxx/core/templates/osmaxx/navigation.html:7 +#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:24 msgid "Existing Excerpt / Country" msgstr "" #: osmaxx/core/templates/osmaxx/navigation.html:12 +#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:31 msgid "New Excerpt" msgstr "" @@ -787,24 +792,24 @@ msgstr "" msgid "Existing excerpts" msgstr "" -#: osmaxx/excerptexport/forms/order_options_mixin.py:12 +#: osmaxx/excerptexport/forms/order_options_mixin.py:11 msgid "GIS export formats" msgstr "" -#: osmaxx/excerptexport/forms/order_options_mixin.py:18 +#: osmaxx/excerptexport/forms/order_options_mixin.py:17 msgid "Coordinate system" msgstr "" -#: osmaxx/excerptexport/forms/order_options_mixin.py:23 +#: osmaxx/excerptexport/forms/order_options_mixin.py:22 msgid "Detail level" msgstr "" -#: osmaxx/excerptexport/forms/order_options_mixin.py:37 +#: osmaxx/excerptexport/forms/order_options_mixin.py:36 msgid "Export options" msgstr "" -#: osmaxx/excerptexport/forms/order_options_mixin.py:44 -msgid "GIS options (ignored for Garmin)" +#: osmaxx/excerptexport/forms/order_options_mixin.py:43 +msgid "GIS options (ignored for Garmin and PBF)" msgstr "" #: osmaxx/excerptexport/models/excerpt.py:20 @@ -863,23 +868,23 @@ msgstr "" msgid "{} But the result file is not available." msgstr "" -#: osmaxx/excerptexport/models/extraction_order.py:15 +#: osmaxx/excerptexport/models/extraction_order.py:16 msgid "CRS" msgstr "" -#: osmaxx/excerptexport/models/extraction_order.py:20 +#: osmaxx/excerptexport/models/extraction_order.py:21 msgid "process link" msgstr "" -#: osmaxx/excerptexport/models/extraction_order.py:21 +#: osmaxx/excerptexport/models/extraction_order.py:22 msgid "orderer" msgstr "" -#: osmaxx/excerptexport/models/extraction_order.py:23 +#: osmaxx/excerptexport/models/extraction_order.py:24 msgid "excerpt" msgstr "" -#: osmaxx/excerptexport/models/extraction_order.py:25 +#: osmaxx/excerptexport/models/extraction_order.py:26 msgid "progress URL" msgstr "" @@ -1009,54 +1014,17 @@ msgstr "" msgid "refresh page to see changes" msgstr "" -#: osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html:8 -msgid "Please Contact Your Administrator or Request Access" -msgstr "" - -#: osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html:10 -msgid "" -"You do not appear to have the proper access rights. Please contact your\n" -" supervisor/administrator and ask for the needed access " -"rights." -msgstr "" - -#: osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html:16 -msgid "" -"If you logged in by a new OpenStreetMap-account, please request its " -"activation: " -msgstr "" - -#: osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html:19 -msgid "Request access" -msgstr "" - -#: osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html:21 -msgid "and try again." -msgstr "" - -#: osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html:24 -msgid "Please Log In" -msgstr "" - -#: osmaxx/excerptexport/templates/excerptexport/templates/access_denied.html:25 -msgid "This site has pages, which are protected. Please login and try again." -msgstr "" - #: osmaxx/excerptexport/templates/excerptexport/templates/index.html:16 msgid "Get the OpenStreetMap data you want in the file format you need" msgstr "" #: osmaxx/excerptexport/templates/excerptexport/templates/index.html:23 -msgid "Export an existing excerpt" -msgstr "" - -#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:26 msgid "" -"(a predefined country or other administrative area,
or one of your " -"previously defined excerpts)" +"Export a predefined country or other administrative area,
or one of " +"your previously defined excerpts" msgstr "" -#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:31 +#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:30 msgid "Define & export a new excerpt" msgstr "" @@ -1072,22 +1040,12 @@ msgstr "" msgid "New excerpt export" msgstr "" -#: osmaxx/excerptexport/views.py:45 +#: osmaxx/excerptexport/views.py:38 #, python-brace-format msgid "Queued extraction order {id}. The conversion process will start soon." msgstr "" -#: osmaxx/excerptexport/views.py:189 osmaxx/excerptexport/views.py:220 -msgid "Sending of access request failed. Please contact an administrator." -msgstr "" - -#: osmaxx/excerptexport/views.py:213 -msgid "" -"Your access request has been sent successfully. You will receive an email " -"when your account is ready." -msgstr "" - -#: osmaxx/excerptexport/views.py:272 +#: osmaxx/excerptexport/views.py:206 msgid "" "Exports are currently running for this excerpt. Please try deleting again, " "when these are finished." @@ -1098,15 +1056,19 @@ msgstr "" msgid "Profile" msgstr "" -#: osmaxx/profile/forms.py:11 +#: osmaxx/profile/email_confirmation.py:49 +msgid "To activate your email, click the link in the confirmation email." +msgstr "" + +#: osmaxx/profile/forms.py:10 msgid "email address" msgstr "" -#: osmaxx/profile/models.py:13 +#: osmaxx/profile/models.py:15 msgid "user" msgstr "" -#: osmaxx/profile/models.py:14 +#: osmaxx/profile/models.py:16 msgid "unverified email" msgstr "" @@ -1132,33 +1094,29 @@ msgstr "" msgid "Confirm your email address" msgstr "" -#: osmaxx/profile/views.py:51 -msgid "To activate your email, click the link in the confirmation email." -msgstr "" - -#: osmaxx/profile/views.py:67 +#: osmaxx/profile/views.py:26 msgid "" "You have not set an email address. You must set a valid email address to use " "OSMaxx." msgstr "" -#: osmaxx/profile/views.py:73 +#: osmaxx/profile/views.py:32 msgid "" "Your email has not been validated. Please check your inbox and validate your " "email address or resend the verification email." msgstr "" -#: osmaxx/profile/views.py:115 +#: osmaxx/profile/views.py:74 msgid "Profile successfully updated" msgstr "" -#: osmaxx/profile/views.py:120 +#: osmaxx/profile/views.py:79 msgid "" "Verification token too old or invalid. Please resend the confirmation email " "and try again." msgstr "" -#: osmaxx/profile/views.py:131 +#: osmaxx/profile/views.py:89 msgid "Successfully verified your email address." msgstr "" @@ -1166,7 +1124,7 @@ msgstr "" msgid "Rest API" msgstr "" -#: osmaxx/utilities/shortcuts.py:43 +#: osmaxx/utils/shortcuts.py:43 msgid "" "There is no email address assigned to your account. You won't be notified by " "email!" diff --git a/osmaxx/profile/email_confirmation.py b/osmaxx/profile/email_confirmation.py new file mode 100644 index 000000000..9c44ccb86 --- /dev/null +++ b/osmaxx/profile/email_confirmation.py @@ -0,0 +1,50 @@ +from django.conf import settings +from django.contrib import messages +from django.core.cache import cache +from django.core.mail import send_mail +from django.template.loader import render_to_string +from django.urls import reverse +from django.utils.text import unescape_entities +from django.utils.translation import ugettext as _ +from furl import furl + +RATE_LIMIT_SECONDS = 30 + + +def send_email_confirmation(profile, request, redirection_target): + user = profile.associated_user + assert user == request.user + if cache.get(user.id): + return + to_email = profile.unverified_email + if to_email: + cache.set(user.id, 'dummy value', timeout=RATE_LIMIT_SECONDS) + user_administrator_email = settings.OSMAXX['ACCOUNT_MANAGER_EMAIL'] + token = profile.activation_key() + f = furl(reverse('profile:activation')) + f.args['token'] = token + if redirection_target: + f.args['next'] = redirection_target + token_url = request.build_absolute_uri(f.url) + subject = render_to_string('profile/verification_email/subject.txt', context={}).strip() + subject = ''.join(subject.splitlines()) + subject = unescape_entities(subject) # HACK: workaround for https://github.com/geometalab/osmaxx/issues/771 + message = render_to_string( + 'profile/verification_email/body.txt', + context=dict( + token_url=token_url, + username=user.username, + new_email_address=to_email, + domain=request.get_host(), + ) + ) + message = unescape_entities(message) # HACK: workaround for https://github.com/geometalab/osmaxx/issues/771 + send_mail( + subject=subject, + message=message, + from_email=user_administrator_email, + recipient_list=[to_email], + ) + messages.add_message( + request, messages.INFO, _('To activate your email, click the link in the confirmation email.') + ) diff --git a/osmaxx/profile/forms.py b/osmaxx/profile/forms.py index 493d88858..a53fe92c4 100644 --- a/osmaxx/profile/forms.py +++ b/osmaxx/profile/forms.py @@ -1,6 +1,5 @@ from crispy_forms.helper import FormHelper from crispy_forms.layout import Submit, Layout -from django.core.urlresolvers import reverse from django import forms from django.utils.translation import ugettext_lazy as _ @@ -13,9 +12,7 @@ class ProfileForm(forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper = FormHelper() - self.helper.form_id = 'id-profileForm' - self.helper.form_method = 'post' - self.helper.form_action = reverse('profile:edit_view') + self.helper.form_tag = False self.helper.add_input(Submit('submit', 'Submit')) self.helper.layout = Layout( 'unverified_email', diff --git a/osmaxx/profile/models.py b/osmaxx/profile/models.py index 76f79a54c..95840eb96 100644 --- a/osmaxx/profile/models.py +++ b/osmaxx/profile/models.py @@ -5,6 +5,8 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ +from osmaxx.profile.email_confirmation import send_email_confirmation + class Profile(models.Model): REGISTRATION_VERIFICATION_TIMEOUT_DAYS = 2 @@ -14,7 +16,7 @@ class Profile(models.Model): unverified_email = models.EmailField(verbose_name=_('unverified email'), max_length=200, null=True) def has_validated_email(self): - return self.associated_user.email == self.unverified_email + return self.unverified_email and self.associated_user.email == self.unverified_email def activation_key(self): key = signing.dumps( @@ -22,6 +24,9 @@ def activation_key(self): salt=self.PROFILE_SALT) return key + def request_email_address_confirmation(self, request, redirection_target): + send_email_confirmation(profile=self, request=request, redirection_target=redirection_target) + def validate_key(self, activation_key): try: return signing.loads( diff --git a/osmaxx/profile/templates/profile/profile_edit.html b/osmaxx/profile/templates/profile/profile_edit.html index 06135d810..b1005411e 100644 --- a/osmaxx/profile/templates/profile/profile_edit.html +++ b/osmaxx/profile/templates/profile/profile_edit.html @@ -11,11 +11,16 @@

{% trans 'Profile for' %} {{ request.user.username }}

- {% crispy form %} +
+ {% crispy form %} + {% if request.GET.next %} + + {% endif %} +
{% if request.user.email != request.user.profile.unverified_email %}
- Resend Confirmation Email + Resend Confirmation Email
{% endif %}
diff --git a/osmaxx/profile/views.py b/osmaxx/profile/views.py index b95a25cf0..c19b853e6 100644 --- a/osmaxx/profile/views.py +++ b/osmaxx/profile/views.py @@ -1,58 +1,17 @@ from django.conf import settings from django.contrib import messages -from django.contrib.auth.models import Group -from django.core.cache import cache -from django.core.mail import send_mail +from django.contrib.auth.mixins import LoginRequiredMixin from django.core.urlresolvers import reverse from django.http import HttpResponseRedirect from django.shortcuts import redirect -from django.template.defaultfilters import urlencode -from django.template.loader import render_to_string from django.utils.translation import ugettext as _ from django.views import generic -from osmaxx.contrib.auth.frontend_permissions import LoginRequiredMixin from osmaxx.profile.forms import ProfileForm from osmaxx.profile.models import Profile -class SendVerificationEmailMixin(object): - RATE_LIMIT_SECONDS = 30 - - def _send_email_verification(self, profile): - if cache.get(profile.associated_user.id): - return - to_email = profile.unverified_email - if to_email: - cache.set(profile.associated_user.id, 'dummy value', timeout=self.RATE_LIMIT_SECONDS) - user_administrator_email = settings.OSMAXX['ACCOUNT_MANAGER_EMAIL'] - token = profile.activation_key() - token_url = '{}?token={}'.format( - self.request.build_absolute_uri(reverse('profile:activation')), urlencode(token) - ) - subject = render_to_string('profile/verification_email/subject.txt', context={}).strip() - subject = ''.join(subject.splitlines()) - message = render_to_string( - 'profile/verification_email/body.txt', - context=dict( - token_url=token_url, - username=self.request.user.username, - new_email_address=to_email, - domain=self.request.get_host(), - ) - ) - send_mail( - subject=subject, - message=message, - from_email=user_administrator_email, - recipient_list=[to_email], - ) - messages.add_message( - self.request, messages.INFO, _('To activate your email, click the link in the confirmation email.') - ) - - -class ProfileView(SendVerificationEmailMixin, LoginRequiredMixin, generic.UpdateView): +class ProfileView(LoginRequiredMixin, generic.UpdateView): form_class = ProfileForm template_name = 'profile/profile_edit.html' @@ -75,7 +34,7 @@ def get(self, *args, **kwargs): ) if self.is_new_user(): self._move_email_from_user_to_profile(user, profile) - self._send_email_verification(profile) + profile.request_email_address_confirmation(self.request, redirection_target=self.request.GET.get('next')) else: self._ensure_profile_has_email(profile, user) user.refresh_from_db() @@ -92,7 +51,7 @@ def post(self, *args, **kwargs): if isinstance(response, HttpResponseRedirect): # successful form validation profile = Profile.objects.get(associated_user=self.request.user) if not profile.has_validated_email(): - self._send_email_verification(profile=profile) + profile.request_email_address_confirmation(self.request, redirection_target=self.request.POST.get('next')) return response def _ensure_profile_has_email(self, profile, user): @@ -116,7 +75,7 @@ def get_success_url(self): return reverse('profile:edit_view') -class ActivationView(SendVerificationEmailMixin, LoginRequiredMixin, generic.UpdateView): +class ActivationView(LoginRequiredMixin, generic.UpdateView): error_msg = _('Verification token too old or invalid. Please resend the confirmation email and try again.') def get(self, request, *args, **kwargs): @@ -127,30 +86,24 @@ def get(self, request, *args, **kwargs): if data: user.email = data['email'] user.save() - self._set_group(user) messages.add_message(self.request, messages.SUCCESS, _('Successfully verified your email address.')) + redirection_target = request.GET.get('next', None) + if redirection_target: + return redirect(redirection_target) else: messages.add_message(self.request, messages.ERROR, self.error_msg) else: messages.add_message(self.request, messages.ERROR, self.error_msg) return redirect(reverse('profile:edit_view')) - def _set_group(self, user): - if settings.REGISTRATION_OPEN and not user_in_frontend_group(user): - self._grant_user_access(user) - - def _grant_user_access(self, user): - group = Group.objects.get(name=settings.OSMAXX_FRONTEND_USER_GROUP) - group.user_set.add(user) - -class ResendVerificationEmail(SendVerificationEmailMixin, LoginRequiredMixin, generic.RedirectView): +class ResendVerificationEmail(LoginRequiredMixin, generic.RedirectView): def get_redirect_url(self, *args, **kwargs): return reverse('profile:edit_view') def get(self, request, *args, **kwargs): profile = Profile.objects.get(associated_user=self.request.user) - self._send_email_verification(profile=profile) + profile.request_email_address_confirmation(self.request, redirection_target=self.request.GET.get('next')) return super().get(request, *args, **kwargs) diff --git a/osmaxx/utilities/model_mixins.py b/osmaxx/utilities/model_mixins.py deleted file mode 100644 index 0e961023e..000000000 --- a/osmaxx/utilities/model_mixins.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.core.urlresolvers import reverse -from django.db import models - - -class AdminUrlModelMixin(models.Model): - class Meta: - abstract = True - - def get_admin_url(self): - admin_view_name = 'admin:{app_label}_{model_name}_change'.format( - app_label=self._meta.app_label, - model_name=self._meta.model_name, - ) - return reverse(admin_view_name, args=(self.id,)) diff --git a/osmaxx/utils/__init__.py b/osmaxx/utils/__init__.py index bacdea616..16fdf78c7 100644 --- a/osmaxx/utils/__init__.py +++ b/osmaxx/utils/__init__.py @@ -1,10 +1,6 @@ -# As `with`-statements are used as if they were functions, let's expose changed_dir under a name -# following function naming conventions (snake_case) rather than class naming conventions (CamelCase): -from osmaxx.utils.directory_changer_helper import changed_dir from .frozendict import frozendict __all__ = [ - 'changed_dir', 'frozendict', ] diff --git a/osmaxx/utilities/dict_helpers.py b/osmaxx/utils/dict_helpers.py similarity index 100% rename from osmaxx/utilities/dict_helpers.py rename to osmaxx/utils/dict_helpers.py diff --git a/osmaxx/utils/directory_changer_helper.py b/osmaxx/utils/directory_changer_helper.py deleted file mode 100644 index 50d1f0516..000000000 --- a/osmaxx/utils/directory_changer_helper.py +++ /dev/null @@ -1,53 +0,0 @@ -import shutil -import os - - -class changed_dir: # pragma: nocover # noqa -> disable=N801 - """ - Changes into a arbitrary directory, switching back to the directory after execution of the with statement. - - directory: - the directory that should be changed into. - create_if_not_exists: - set this to False, if the chgdir should fail loudly. If set to `True`, - tries to create the directory if it doesn't exist - - fallback_to_tempdir_as_last_resort: - assumes create_if_not_exists is also True - creates a tmp directory as last resort, and does work there. Defaults to False. - """ - - def __init__(self, directory, create_if_not_exists=True): - self.old_dir = os.getcwd() - self.new_dir = directory - self.success = None - self.create_if_not_exists = create_if_not_exists - - def __enter__(self): - try: - try: - # see if we can enter the directory - os.chdir(self.new_dir) - self.success = True - except OSError: - if self.create_if_not_exists: - try: - # see if we can create it - os.mkdir(self.new_dir) - os.chdir(self.new_dir) - self.success = True - except OSError: - raise - else: - raise - finally: - return self - - def __exit__(self, type, value, traceback): - os.chdir(self.old_dir) - try: - if not self.success: - shutil.rmtree(self.new_dir) - except: - pass - return isinstance(value, OSError) diff --git a/osmaxx/utilities/shortcuts.py b/osmaxx/utils/shortcuts.py similarity index 100% rename from osmaxx/utilities/shortcuts.py rename to osmaxx/utils/shortcuts.py diff --git a/osmaxx_conversion_service/Procfile.mediator.dev b/osmaxx_conversion_service/Procfile.mediator.dev deleted file mode 100644 index 9f2921876..000000000 --- a/osmaxx_conversion_service/Procfile.mediator.dev +++ /dev/null @@ -1,2 +0,0 @@ -mediator: python3 ./osmaxx_conversion_service/manage.py runserver_plus ${APP_HOST}:${APP_PORT} -harvester: python3 ./osmaxx_conversion_service/manage.py result_harvester diff --git a/osmaxx_conversion_service/Procfile.mediator.prod b/osmaxx_conversion_service/Procfile.mediator.prod deleted file mode 100644 index 8b6b06dd8..000000000 --- a/osmaxx_conversion_service/Procfile.mediator.prod +++ /dev/null @@ -1,2 +0,0 @@ -mediator: gunicorn --workers ${NUM_WORKERS} osmaxx_conversion_service.config.wsgi --bind ${APP_HOST}:${APP_PORT} -harvester: python3 ./osmaxx_conversion_service/manage.py result_harvester diff --git a/osmaxx_conversion_service/Procfile.worker b/osmaxx_conversion_service/Procfile.worker deleted file mode 100644 index 80ed04830..000000000 --- a/osmaxx_conversion_service/Procfile.worker +++ /dev/null @@ -1 +0,0 @@ -worker: ${HOME}/entrypoint/wait-for-it.sh localhost:5432 -t 30 && python3 ./osmaxx_conversion_service/manage.py rqworker ${WORKER_QUEUES:-default high} diff --git a/requirements-all.txt b/requirements-all.txt index 994e58e5b..35cc165fb 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -4,114 +4,114 @@ # # pip-compile --output-file requirements-all.txt requirements-all.in # -astroid==1.4.9 # via pylint, pylint-celery, pylint-flask, pylint-plugin-utils, requirements-detector +astroid==1.5.2 # via pylint, pylint-celery, pylint-flask, pylint-plugin-utils, requirements-detector click==6.7 # via mkdocs, rq -contextlib2==0.5.4 # via raven +contextlib2==0.5.5 # via raven coverage==4.3.4 decorator==4.0.11 # via ipython, traitlets -defusedxml==0.4.1 # via python3-openid +defusedxml==0.5.0 # via python3-openid, social-auth-core django-crispy-forms==1.6.1 -django-debug-toolbar==1.6 +django-debug-toolbar==1.7 django-downloadview==1.9 -django-environ==0.4.1 -django-extensions==1.7.5 -django-filter==1.0.1 -django-model-utils==2.6.1 -django-rq==0.9.4 +django-environ==0.4.3 +django-extensions==1.7.8 +django-filter==1.0.2 +django-model-utils==3.0.0 +django-rq==0.9.5 django-secure==1.0.1 django-stored-messages==1.4.0 -Django==1.10.5 # via django-debug-toolbar, django-downloadview, django-environ, django-model-utils, django-rq, django-secure, django-stored-messages, geometalab.drf-utm-zone-info, geometalab.osm-pbf-file-size-estimation-service +django==1.10.7 # via django-debug-toolbar, django-downloadview, django-environ, django-model-utils, django-rq, django-secure, django-stored-messages, geometalab.drf-utm-zone-info, geometalab.osm-pbf-file-size-estimation-service djangorestframework-gis==0.10.1 -djangorestframework-jwt==1.9.0 +djangorestframework-jwt==1.10.0 djangorestframework==3.4.7 docutils==0.13.1 # via pyroma dodgy==0.1.9 # via prospector drf-extensions==0.3.1 -flake8==3.2.1 -GeoAlchemy2==0.4.0 +flake8==3.3.0 +furl==1.0.0 +geoalchemy2==0.4.0 geometalab.drf-utm-zone-info==0.2.0 geometalab.osm-pbf-file-size-estimation-service==1.1.0 gevent==1.2.1 -greenlet==0.4.11 # via gevent -gunicorn==19.6.0 -ipython-genutils==0.1.0 # via traitlets -ipython==5.1.0 +greenlet==0.4.12 # via gevent +gunicorn==19.7.1 +ipython-genutils==0.2.0 # via traitlets +ipython==6.0.0 isort==4.2.5 # via pylint -Jinja2==2.9.4 # via mkdocs -lazy-object-proxy==1.2.2 # via astroid +jedi==0.10.2 # via ipython +jinja2==2.9.6 # via mkdocs +lazy-object-proxy==1.3.0 # via astroid livereload==2.5.1 # via mkdocs -markdown==2.6.7 -MarkupSafe==0.23 # via jinja2 -mccabe==0.5.3 # via flake8, prospector, pylint -mkdocs==0.16.1 +markdown==2.6.8 +markupsafe==1.0 # via jinja2 +mccabe==0.6.1 # via flake8, prospector, pylint +memoize==1.0.0 +mkdocs==0.16.3 multidict==2.1.4 # via yarl -numpy==1.12.0 -oauthlib==2.0.1 # via requests-oauthlib, social-auth-core +numpy==1.12.1 +oauthlib==2.0.2 # via requests-oauthlib, social-auth-core +orderedmultidict==0.7.11 # via furl pep8-naming==0.4.1 # via prospector pexpect==4.2.1 # via ipython pickleshare==0.7.4 # via ipython -prompt-toolkit==1.0.9 # via ipython -prospector==0.12.4 -psycopg2==2.6.2 +prompt-toolkit==1.0.14 # via ipython +prospector==0.12.5 +psycopg2==2.7.1 ptyprocess==0.5.1 # via pexpect -py==1.4.32 # via pytest +py==1.4.33 # via pytest pycodestyle==2.0.0 # via flake8, prospector pydocstyle==1.0.0 # via prospector -pyflakes==1.3.0 # via flake8, prospector -pygments==2.1.3 # via ipython +pyflakes==1.5.0 # via flake8, prospector +pygments==2.2.0 # via ipython pyhamcrest==1.9.0 -PyJWT==1.4.2 # via djangorestframework-jwt, social-auth-core +pyjwt==1.5.0 # via djangorestframework-jwt, social-auth-core pylint-celery==0.3 # via prospector -pylint-common==0.2.2 # via prospector +pylint-common==0.2.5 # via prospector pylint-django==0.7.2 # via prospector pylint-flask==0.5 # via prospector -pylint-plugin-utils==0.2.4 # via prospector, pylint-celery, pylint-django, pylint-flask -pylint==1.6.4 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask, pylint-plugin-utils -pypandoc==1.3.3 +pylint-plugin-utils==0.2.6 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask +pylint==1.7.1 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask, pylint-plugin-utils +pypandoc==1.4 pyroma==2.2 -pytest-base-url==1.2.0 # via pytest-selenium +pytest-base-url==1.3.0 # via pytest-selenium pytest-cov==2.4.0 pytest-django==3.1.2 -pytest-html==1.13.0 # via pytest-selenium -pytest-mock==1.5.0 -pytest-selenium==1.7.0 -pytest-variables==1.4 # via pytest-selenium -pytest==3.0.5 # via pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-mock, pytest-selenium, pytest-variables -python-social-auth[django]==0.3.3 -python3-openid==3.0.10 # via social-auth-core -PyYAML==3.12 # via mkdocs, prospector, vcrpy -raven==5.32.0 +pytest-html==1.14.2 # via pytest-selenium +pytest-metadata==1.3.0 # via pytest-html +pytest-mock==1.6.0 +pytest-selenium==1.10.0 +pytest-variables==1.6.1 # via pytest-selenium +pytest==3.0.7 # via pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-metadata, pytest-mock, pytest-selenium, pytest-variables +python-social-auth[django]==0.3.6 +python3-openid==3.1.0 # via social-auth-core +pyyaml==3.12 # via mkdocs, prospector, vcrpy +raven==6.0.0 redis==2.10.5 # via rq -requests-mock==1.2.0 -requests-oauthlib==0.7.0 # via social-auth-core -requests==2.12.5 +requests-mock==1.3.0 +requests-oauthlib==0.8.0 # via social-auth-core +requests==2.13.0 requirements-detector==0.5.2 # via prospector rq==0.7.1 -ruamel.yaml==0.13.9 -scandir==1.4 -scipy==0.18.1 -selenium==3.0.2 # via pytest-selenium +ruamel.yaml==0.14.11 +scandir==1.5 +scipy==0.19.0 +selenium==3.4.1 # via pytest-selenium setoptconf==0.2.0 # via prospector simplegeneric==0.8.1 # via ipython -six==1.10.0 # via astroid, django-downloadview, django-environ, django-extensions, livereload, prompt-toolkit, pyhamcrest, pylint, requests-mock, social-auth-app-django, social-auth-core, sqlalchemy-utils, traitlets, vcrpy -social-auth-app-django==0.1.0 # via python-social-auth -social-auth-core==0.2.1 # via python-social-auth, social-auth-app-django -SQLAlchemy-Utils==0.32.12 +six==1.10.0 # via astroid, django-downloadview, django-environ, django-extensions, furl, livereload, orderedmultidict, prompt-toolkit, pyhamcrest, pylint, requests-mock, social-auth-app-django, social-auth-core, sqlalchemy-utils, traitlets, vcrpy +social-auth-app-django==1.1.0 # via python-social-auth +social-auth-core==1.2.0 # via python-social-auth, social-auth-app-django +sqlalchemy-utils==0.32.14 sqlalchemy-views==0.2.1 -SQLAlchemy==1.1.5 # via geoalchemy2, sqlalchemy-utils, sqlalchemy-views -sqlparse==0.2.2 # via django-debug-toolbar -tornado==4.4.2 # via livereload, mkdocs -traitlets==4.3.1 # via ipython -vcrpy==1.10.5 -vulture==0.12 +sqlalchemy==1.1.9 # via geoalchemy2, sqlalchemy-utils, sqlalchemy-views +sqlparse==0.2.3 # via django-debug-toolbar +tornado==4.5.1 # via livereload, mkdocs +traitlets==4.3.2 # via ipython +vcrpy==1.11.0 +vulture==0.14 wcwidth==0.1.7 # via prompt-toolkit -Werkzeug==0.11.15 +werkzeug==0.12.1 wheel==0.29.0 -whitenoise==3.2.3 -wrapt==1.10.8 # via astroid, vcrpy -yarl==0.8.1 # via vcrpy - -# The following packages are commented out because they are -# considered to be unsafe in a requirements file: -# pip # via pypandoc -# setuptools # via django-downloadview, ipython, pyhamcrest, pypandoc, pyroma +whitenoise==3.3.0 +wrapt==1.10.10 # via astroid, vcrpy +yarl==0.10.1 # via vcrpy diff --git a/requirements.in b/requirements.in index 2ca4870e8..581a74af2 100644 --- a/requirements.in +++ b/requirements.in @@ -11,6 +11,7 @@ geometalab.drf-utm-zone-info>=0.1.0,<1.0.0 djangorestframework-jwt requests markdown +memoize django-filter rq django-rq @@ -29,6 +30,7 @@ SQLAlchemy SQLAlchemy-Utils sqlalchemy-views GeoAlchemy2 +furl # fallback for python < 3.5 scandir diff --git a/requirements.txt b/requirements.txt index 3861a2d55..41883af75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,111 +4,112 @@ # # pip-compile --output-file requirements.txt requirements.in # -astroid==1.4.9 # via pylint, pylint-celery, pylint-flask, pylint-plugin-utils, requirements-detector +astroid==1.5.2 # via pylint, pylint-celery, pylint-flask, pylint-plugin-utils, requirements-detector click==6.7 # via mkdocs, rq -contextlib2==0.5.4 # via raven +contextlib2==0.5.5 # via raven coverage==4.3.4 decorator==4.0.11 # via ipython, traitlets -defusedxml==0.4.1 # via python3-openid +defusedxml==0.5.0 # via python3-openid, social-auth-core django-crispy-forms==1.6.1 -django-debug-toolbar==1.6 +django-debug-toolbar==1.7 django-downloadview==1.9 -django-environ==0.4.1 -django-extensions==1.7.5 -django-filter==1.0.1 -django-model-utils==2.6.1 -django-rq==0.9.4 +django-environ==0.4.3 +django-extensions==1.7.8 +django-filter==1.0.2 +django-model-utils==3.0.0 +django-rq==0.9.5 django-secure==1.0.1 django-stored-messages==1.4.0 -Django==1.10.5 # via django-debug-toolbar, django-downloadview, django-environ, django-model-utils, django-rq, django-secure, django-stored-messages, geometalab.drf-utm-zone-info, geometalab.osm-pbf-file-size-estimation-service +django==1.10.7 # via django-debug-toolbar, django-downloadview, django-environ, django-model-utils, django-rq, django-secure, django-stored-messages, geometalab.drf-utm-zone-info, geometalab.osm-pbf-file-size-estimation-service djangorestframework-gis==0.10.1 -djangorestframework-jwt==1.9.0 +djangorestframework-jwt==1.10.0 djangorestframework==3.4.7 docutils==0.13.1 # via pyroma dodgy==0.1.9 # via prospector drf-extensions==0.3.1 -flake8==3.2.1 -GeoAlchemy2==0.4.0 +flake8==3.3.0 +furl==1.0.0 +geoalchemy2==0.4.0 geometalab.drf-utm-zone-info==0.2.0 geometalab.osm-pbf-file-size-estimation-service==1.1.0 gevent==1.2.1 -greenlet==0.4.11 # via gevent -gunicorn==19.6.0 -ipython-genutils==0.1.0 # via traitlets -ipython==5.1.0 +greenlet==0.4.12 # via gevent +gunicorn==19.7.1 +ipython-genutils==0.2.0 # via traitlets +ipython==6.0.0 isort==4.2.5 # via pylint -Jinja2==2.9.4 # via mkdocs -lazy-object-proxy==1.2.2 # via astroid +jedi==0.10.2 # via ipython +jinja2==2.9.6 # via mkdocs +lazy-object-proxy==1.3.0 # via astroid livereload==2.5.1 # via mkdocs -markdown==2.6.7 -MarkupSafe==0.23 # via jinja2 -mccabe==0.5.3 # via flake8, prospector, pylint -mkdocs==0.16.1 +markdown==2.6.8 +markupsafe==1.0 # via jinja2 +mccabe==0.6.1 # via flake8, prospector, pylint +memoize==1.0.0 +mkdocs==0.16.3 multidict==2.1.4 # via yarl -numpy==1.12.0 -oauthlib==2.0.1 # via requests-oauthlib, social-auth-core +numpy==1.12.1 +oauthlib==2.0.2 # via requests-oauthlib, social-auth-core +orderedmultidict==0.7.11 # via furl pep8-naming==0.4.1 # via prospector pexpect==4.2.1 # via ipython pickleshare==0.7.4 # via ipython -prompt-toolkit==1.0.9 # via ipython -prospector==0.12.4 -psycopg2==2.6.2 +prompt-toolkit==1.0.14 # via ipython +prospector==0.12.5 +psycopg2==2.7.1 ptyprocess==0.5.1 # via pexpect -py==1.4.32 # via pytest +py==1.4.33 # via pytest pycodestyle==2.0.0 # via flake8, prospector pydocstyle==1.0.0 # via prospector -pyflakes==1.3.0 # via flake8, prospector -pygments==2.1.3 # via ipython +pyflakes==1.5.0 # via flake8, prospector +pygments==2.2.0 # via ipython pyhamcrest==1.9.0 -PyJWT==1.4.2 # via djangorestframework-jwt, social-auth-core +pyjwt==1.5.0 # via djangorestframework-jwt, social-auth-core pylint-celery==0.3 # via prospector -pylint-common==0.2.2 # via prospector +pylint-common==0.2.5 # via prospector pylint-django==0.7.2 # via prospector pylint-flask==0.5 # via prospector -pylint-plugin-utils==0.2.4 # via prospector, pylint-celery, pylint-django, pylint-flask -pylint==1.6.4 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask, pylint-plugin-utils +pylint-plugin-utils==0.2.6 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask +pylint==1.7.1 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask, pylint-plugin-utils pyroma==2.2 -pytest-base-url==1.2.0 # via pytest-selenium +pytest-base-url==1.3.0 # via pytest-selenium pytest-cov==2.4.0 pytest-django==3.1.2 -pytest-html==1.13.0 # via pytest-selenium -pytest-mock==1.5.0 -pytest-selenium==1.7.0 -pytest-variables==1.4 # via pytest-selenium -pytest==3.0.5 # via pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-mock, pytest-selenium, pytest-variables -python-social-auth[django]==0.3.3 -python3-openid==3.0.10 # via social-auth-core -PyYAML==3.12 # via mkdocs, prospector, vcrpy -raven==5.32.0 +pytest-html==1.14.2 # via pytest-selenium +pytest-metadata==1.3.0 # via pytest-html +pytest-mock==1.6.0 +pytest-selenium==1.10.0 +pytest-variables==1.6.1 # via pytest-selenium +pytest==3.0.7 # via pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-metadata, pytest-mock, pytest-selenium, pytest-variables +python-social-auth[django]==0.3.6 +python3-openid==3.1.0 # via social-auth-core +pyyaml==3.12 # via mkdocs, prospector, vcrpy +raven==6.0.0 redis==2.10.5 # via rq -requests-mock==1.2.0 -requests-oauthlib==0.7.0 # via social-auth-core -requests==2.12.5 +requests-mock==1.3.0 +requests-oauthlib==0.8.0 # via social-auth-core +requests==2.13.0 requirements-detector==0.5.2 # via prospector rq==0.7.1 -scandir==1.4 -scipy==0.18.1 -selenium==3.0.2 # via pytest-selenium +scandir==1.5 +scipy==0.19.0 +selenium==3.4.1 # via pytest-selenium setoptconf==0.2.0 # via prospector simplegeneric==0.8.1 # via ipython -six==1.10.0 # via astroid, django-downloadview, django-environ, django-extensions, livereload, prompt-toolkit, pyhamcrest, pylint, requests-mock, social-auth-app-django, social-auth-core, sqlalchemy-utils, traitlets, vcrpy -social-auth-app-django==0.1.0 # via python-social-auth -social-auth-core==0.2.1 # via python-social-auth, social-auth-app-django -SQLAlchemy-Utils==0.32.12 +six==1.10.0 # via astroid, django-downloadview, django-environ, django-extensions, furl, livereload, orderedmultidict, prompt-toolkit, pyhamcrest, pylint, requests-mock, social-auth-app-django, social-auth-core, sqlalchemy-utils, traitlets, vcrpy +social-auth-app-django==1.1.0 # via python-social-auth +social-auth-core==1.2.0 # via python-social-auth, social-auth-app-django +sqlalchemy-utils==0.32.14 sqlalchemy-views==0.2.1 -SQLAlchemy==1.1.5 # via geoalchemy2, sqlalchemy-utils, sqlalchemy-views -sqlparse==0.2.2 # via django-debug-toolbar -tornado==4.4.2 # via livereload, mkdocs -traitlets==4.3.1 # via ipython -vcrpy==1.10.5 -vulture==0.12 +sqlalchemy==1.1.9 # via geoalchemy2, sqlalchemy-utils, sqlalchemy-views +sqlparse==0.2.3 # via django-debug-toolbar +tornado==4.5.1 # via livereload, mkdocs +traitlets==4.3.2 # via ipython +vcrpy==1.11.0 +vulture==0.14 wcwidth==0.1.7 # via prompt-toolkit -Werkzeug==0.11.15 +werkzeug==0.12.1 wheel==0.29.0 -whitenoise==3.2.3 -wrapt==1.10.8 # via astroid, vcrpy -yarl==0.8.1 # via vcrpy - -# The following packages are commented out because they are -# considered to be unsafe in a requirements file: -# setuptools # via django-downloadview, ipython, pyhamcrest, pyroma +whitenoise==3.3.0 +wrapt==1.10.10 # via astroid, vcrpy +yarl==0.10.1 # via vcrpy diff --git a/tests/conftest.py b/tests/conftest.py index 97e2b9a6a..65b199bbf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -100,9 +100,6 @@ def pytest_configure(): 'osmaxx.excerptexport', 'osmaxx.job_progress', 'osmaxx.profile', - - # special model for testing only - 'tests.utilities.test_models', ), PASSWORD_HASHERS=( 'django.contrib.auth.hashers.SHA1PasswordHasher', @@ -154,8 +151,6 @@ def pytest_configure(): OSMAXX={ 'download_file_name': '%(excerpt_name)s-%(date)s.%(content_type)s.%(file_extension)s', 'EXTRACTION_PROCESSING_TIMEOUT_TIMEDELTA': timedelta(hours=48), - # The email adress of this user will be used to generate the mailto link for users - # to request access to osmaxx (access_denied page) 'CONVERSION_SERVICE_URL': 'http://localhost:8901/api/', 'CONVERSION_SERVICE_USERNAME': 'dev', 'CONVERSION_SERVICE_PASSWORD': 'dev', @@ -163,7 +158,6 @@ def pytest_configure(): 'ACCOUNT_MANAGER_EMAIL': 'accountmanager@example.com', }, OSMAXX_FRONTEND_USER_GROUP='osmaxx_frontend_users', - REGISTRATION_OPEN=True, CACHES={ 'default': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', @@ -257,11 +251,9 @@ def authenticated_api_client(api_client, user): @pytest.fixture def frontend_accessible_authenticated_api_client(api_client, user): - from django.conf import settings - from django.contrib.auth.models import Group + from osmaxx.profile.models import Profile - group = Group.objects.get(name=settings.OSMAXX_FRONTEND_USER_GROUP) - user.groups.add(group) + Profile.objects.create(associated_user=user, unverified_email=user.email) return authenticated_client(api_client, user) diff --git a/tests/contrib/auth/test_frontend_permissions.py b/tests/contrib/auth/test_frontend_permissions.py index 11cbfe33e..fe0dba2c2 100644 --- a/tests/contrib/auth/test_frontend_permissions.py +++ b/tests/contrib/auth/test_frontend_permissions.py @@ -1,29 +1,28 @@ -from django.conf import settings -from django.test import TestCase -from django.contrib.auth.models import User, Group -from osmaxx.contrib.auth.frontend_permissions import _may_user_access_osmaxx_frontend - - -class TestFrontendPermissions(TestCase): - def test_user_can_not_access_frontend_by_default(self): - """ - Newly created users (self sign-up) can't do anything that non-authenticated visitors couldn't do, too, so that - we don't have to restrict who can sign up. - """ - a_user = User.objects.create_user('U. Ser', 'user@example.com', 'password') - self.assertFalse(_may_user_access_osmaxx_frontend(a_user)) - - def test_user_can_access_frontend_when_in_osmaxx_group(self): - """ - To activate a user for frontend access, add it to the osmaxx frontend group. - """ - a_user = User.objects.create_user('U. Ser', 'user@example.com', 'password') - a_user.groups.add(Group.objects.get(name=settings.OSMAXX_FRONTEND_USER_GROUP)) - self.assertTrue(_may_user_access_osmaxx_frontend(a_user)) - - def test_superuser_can_access_frontend_even_if_not_in_osmaxx_group(self): - """ - Superusers cannot be created by anonymous self sign-up, so we don't require explicit group membership for them. - """ - an_admin = User.objects.create_superuser('A. D. Min', 'admin@example.com', 'password') - self.assertTrue(_may_user_access_osmaxx_frontend(an_admin)) +import pytest +from django.contrib.auth.models import User +from osmaxx.contrib.auth.frontend_permissions import _user_has_validated_email +from osmaxx.profile.models import Profile + + +def test_new_user_has_no_validated_email_address(db): + a_user = User.objects.create_user('U. Ser', 'user@example.com', 'password') + assert not _user_has_validated_email(a_user) + + +def test_user_with_same_address_on_profile_has_validated_email_address(db): + a_user = User.objects.create_user('U. Ser', 'user@example.com', 'password') + Profile.objects.create(associated_user=a_user, unverified_email=a_user.email) + assert _user_has_validated_email(a_user) + + +def test_new_superuser_has_no_validated_email_address(db): + an_admin = User.objects.create_superuser('A. D. Min', 'admin@example.com', 'password') + assert not _user_has_validated_email(an_admin) + + +@pytest.mark.parametrize('user_email', ['', None]) +@pytest.mark.parametrize('profile_email', ['', None]) +def test_user_with_empty_email_addresses(db, user_email, profile_email): + a_user = User.objects.create_user('U. Ser', user_email, 'password') + Profile.objects.create(associated_user=a_user, unverified_email=profile_email) + assert not _user_has_validated_email(a_user) diff --git a/tests/conversion/conftest.py b/tests/conversion/conftest.py index efb27e59e..1d7b13940 100644 --- a/tests/conversion/conftest.py +++ b/tests/conversion/conftest.py @@ -5,12 +5,12 @@ import pytest from django.conf import settings -import osmaxx.conversion_api.formats -from osmaxx.conversion.converters import detail_levels -from osmaxx.conversion_api import coordinate_reference_systems as crs -from osmaxx.conversion_api.statuses import STARTED, FAILED, FINISHED +from osmaxx.conversion import output_format +from osmaxx.conversion.converters.converter_gis import detail_levels +from osmaxx.conversion.constants import coordinate_reference_systems as crs +from osmaxx.conversion.constants.statuses import STARTED, FAILED, FINISHED -format_list = osmaxx.conversion_api.formats.FORMAT_DEFINITIONS.keys() +format_list = output_format.DEFINITIONS.keys() @pytest.fixture(params=format_list) diff --git a/tests/conversion/converters/bootstrap_test.py b/tests/conversion/converters/bootstrap_test.py index 58fc07229..094606a95 100644 --- a/tests/conversion/converters/bootstrap_test.py +++ b/tests/conversion/converters/bootstrap_test.py @@ -1,9 +1,8 @@ from unittest import mock -from osmaxx.conversion.converters.detail_levels import DETAIL_LEVEL_REDUCED -from tests.conftest import area_polyfile_string - from osmaxx.conversion.converters.converter_gis.bootstrap import bootstrap +from osmaxx.conversion.converters.converter_gis.detail_levels import DETAIL_LEVEL_REDUCED +from tests.conftest import area_polyfile_string def test_filter_scripts_are_executed_in_correct_order(sql_scripts_filter): diff --git a/tests/conversion/converters/converter_test.py b/tests/conversion/converters/converter_test.py index 4014d30e0..4d90324f6 100644 --- a/tests/conversion/converters/converter_test.py +++ b/tests/conversion/converters/converter_test.py @@ -1,10 +1,11 @@ -from osmaxx.conversion.converters.converter import Conversion, convert +from osmaxx.conversion.converters.converter import convert -def test_start_format_extraction(conversion_format, area_name, simple_osmosis_line_string, output_zip_file_path, filename_prefix, detail_level, out_srs, mocker): - gis_converter_mock_create = mocker.patch('osmaxx.conversion.converters.converter_gis.gis.GISConverter.create_gis_export') - garmin_converter_mock_create = mocker.patch('osmaxx.conversion.converters.converter_garmin.garmin.Garmin.create_garmin_export') - conversion = Conversion( +def test_calls_converter_and_returns_none_when_use_worker_is_omitted(conversion_format, area_name, simple_osmosis_line_string, output_zip_file_path, filename_prefix, detail_level, out_srs, mocker): + gis_converter_mock_create = mocker.patch('osmaxx.conversion.converters.converter_gis.perform_export', autospec=True) + garmin_converter_mock_create = mocker.patch('osmaxx.conversion.converters.converter_garmin.perform_export', autospec=True) + pbf_converter_mock_create = mocker.patch('osmaxx.conversion.converters.converter.converter_pbf.perform_export', autospec=True) + convert_return_value = convert( conversion_format=conversion_format, area_name=area_name, osmosis_polygon_file_string=simple_osmosis_line_string, @@ -13,8 +14,8 @@ def test_start_format_extraction(conversion_format, area_name, simple_osmosis_li detail_level=detail_level, out_srs='EPSG:{}'.format(out_srs), ) - conversion.start_format_extraction() - assert gis_converter_mock_create.call_count + garmin_converter_mock_create.call_count == 1 + assert gis_converter_mock_create.call_count + garmin_converter_mock_create.call_count + pbf_converter_mock_create.call_count == 1 + assert convert_return_value is None def test_convert_returns_id_when_use_worker_is_true(conversion_format, area_name, simple_osmosis_line_string, output_zip_file_path, filename_prefix, detail_level, out_srs, rq_mock_return, mocker, monkeypatch): @@ -31,19 +32,3 @@ def test_convert_returns_id_when_use_worker_is_true(conversion_format, area_name use_worker=True, ) assert convert_return_value == 42 - - -def test_convert_starts_conversion(conversion_format, area_name, simple_osmosis_line_string, output_zip_file_path, filename_prefix, detail_level, out_srs, mocker, monkeypatch): - conversion_start_start_format_extraction_mock = mocker.patch('osmaxx.conversion.converters.converter.Conversion') - convert_return_value = convert( - conversion_format=conversion_format, - area_name=area_name, - osmosis_polygon_file_string=simple_osmosis_line_string, - output_zip_file_path=output_zip_file_path, - filename_prefix=filename_prefix, - detail_level=detail_level, - out_srs='EPSG:{}'.format(out_srs), - use_worker=False, - ) - assert convert_return_value is None - assert conversion_start_start_format_extraction_mock.call_count == 1 diff --git a/tests/conversion/converters/garmin_test.py b/tests/conversion/converters/garmin_test.py index 90d408602..a3b2deda3 100644 --- a/tests/conversion/converters/garmin_test.py +++ b/tests/conversion/converters/garmin_test.py @@ -31,6 +31,6 @@ def test_libraries_are_contained_in_source(library_path): def test_create_garmin_export_calls_(output_zip_file_path, area_name, simple_osmosis_line_string, mocker): subprocess_mock = mocker.patch('subprocess.check_call') _create_zip_mock = mocker.patch('osmaxx.conversion.converters.converter_garmin.garmin.Garmin._create_zip') - Garmin(out_zip_file_path=output_zip_file_path, area_name=area_name, polyfile_string=simple_osmosis_line_string).create_garmin_export() - assert 2 == subprocess_mock.call_count + Garmin(output_zip_file_path=output_zip_file_path, area_name=area_name, polyfile_string=simple_osmosis_line_string).create_garmin_export() + assert 3 == subprocess_mock.call_count # 2 calls from garmin and one from the pbf cutter assert 1 == _create_zip_mock.call_count diff --git a/tests/conversion/converters/inside_worker_test/conftest.py b/tests/conversion/converters/inside_worker_test/conftest.py index c9b2314ba..ae9f48d42 100644 --- a/tests/conversion/converters/inside_worker_test/conftest.py +++ b/tests/conversion/converters/inside_worker_test/conftest.py @@ -171,8 +171,8 @@ def _setup_db_functions(self): @contextmanager def import_data(data): - from osmaxx.conversion.converters.converter_gis.bootstrap import bootstrap - mocker.patch.object(bootstrap, 'cut_area_from_pbf', return_value=None) + from osmaxx.conversion.converters.converter_pbf import to_pbf + mocker.patch.object(to_pbf, 'cut_area_from_pbf', return_value=None) bootstrapper = _BootStrapperWithoutPbfFile(data) try: bootstrapper.bootstrap() diff --git a/tests/conversion/converters/inside_worker_test/osm_cutter_test.py b/tests/conversion/converters/inside_worker_test/osm_cutter_test.py index 63bff0d40..46d353275 100644 --- a/tests/conversion/converters/inside_worker_test/osm_cutter_test.py +++ b/tests/conversion/converters/inside_worker_test/osm_cutter_test.py @@ -1,4 +1,4 @@ -from osmaxx.conversion.converters.converter_gis.bootstrap.bootstrap import cut_area_from_pbf +from osmaxx.conversion.converters.converter_pbf.to_pbf import cut_area_from_pbf def test_cut_area_from_pbf(mocker): diff --git a/tests/conversion/management/commands/result_harvester_test.py b/tests/conversion/management/commands/result_harvester_test.py index 1f6be851f..342babc78 100644 --- a/tests/conversion/management/commands/result_harvester_test.py +++ b/tests/conversion/management/commands/result_harvester_test.py @@ -3,7 +3,7 @@ import pytest -from osmaxx.conversion_api.statuses import STARTED +from osmaxx.conversion.constants.statuses import STARTED @pytest.fixture diff --git a/tests/conversion/model_test.py b/tests/conversion/model_test.py index 62711ef05..9be5de96f 100644 --- a/tests/conversion/model_test.py +++ b/tests/conversion/model_test.py @@ -2,7 +2,7 @@ import pytest -from osmaxx.conversion_api.statuses import STARTED, FAILED, FINISHED +from osmaxx.conversion.constants.statuses import STARTED, FAILED, FINISHED @pytest.mark.django_db() diff --git a/tests/conversion/size_estimator_test.py b/tests/conversion/size_estimator_test.py index e95437e2a..c382e5440 100644 --- a/tests/conversion/size_estimator_test.py +++ b/tests/conversion/size_estimator_test.py @@ -1,27 +1,31 @@ import pytest -from osmaxx.conversion.converters import detail_levels +from osmaxx.conversion.converters.converter_gis import detail_levels from osmaxx.conversion.size_estimator import size_estimation_for_format -from osmaxx.conversion_api import formats +from osmaxx.conversion import output_format range_for_format_and_level = { - formats.GARMIN: { + output_format.GARMIN: { detail_levels.DETAIL_LEVEL_ALL: {'upper': 1.4, 'lower': 0.2}, detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 1.4, 'lower': 0.2}, }, - formats.FGDB: { + output_format.PBF: { + detail_levels.DETAIL_LEVEL_ALL: {'upper': 1.000001, 'lower': 0.99999}, + detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 1.000001, 'lower': 0.99999}, + }, + output_format.FGDB: { detail_levels.DETAIL_LEVEL_ALL: {'upper': 7.4, 'lower': 1.6}, detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 2.3, 'lower': 0.6}, }, - formats.GPKG: { + output_format.GPKG: { detail_levels.DETAIL_LEVEL_ALL: {'upper': 17, 'lower': 3.5}, detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 4.9, 'lower': 1.3}, }, - formats.SHAPEFILE: { + output_format.SHAPEFILE: { detail_levels.DETAIL_LEVEL_ALL: {'upper': 45, 'lower': 10}, detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 11, 'lower': 3}, }, - formats.SPATIALITE: { + output_format.SPATIALITE: { detail_levels.DETAIL_LEVEL_ALL: {'upper': 48, 'lower': 4}, detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 16, 'lower': 1.5}, }, diff --git a/tests/conversion/view_test.py b/tests/conversion/view_test.py index a7888ae6b..73548e999 100644 --- a/tests/conversion/view_test.py +++ b/tests/conversion/view_test.py @@ -1,7 +1,7 @@ import pytest from rest_framework.reverse import reverse -from osmaxx.conversion_api.statuses import RECEIVED +from osmaxx.conversion.constants.statuses import RECEIVED authenticated_access_urls = [ reverse('clipping_area-list'), diff --git a/tests/utilities/test_models/migrations/__init__.py b/tests/core/__init__.py similarity index 100% rename from tests/utilities/test_models/migrations/__init__.py rename to tests/core/__init__.py diff --git a/tests/core/templatetags/__init__.py b/tests/core/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/templatetags/test_navigation.py b/tests/core/templatetags/test_navigation.py new file mode 100644 index 000000000..464563d00 --- /dev/null +++ b/tests/core/templatetags/test_navigation.py @@ -0,0 +1,153 @@ +import logging + +import pytest +from django.test import override_settings + +from osmaxx.core.templatetags.navigation import siteabsoluteurl, logger + + +@override_settings( + OSMAXX=dict( + ) +) +def test_siteabsoluteurl_without_secured_proxy_adds_scheme_and_netloc_and_path_prefix(rf, log_warning_mock): + relative_url = 'foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(relative_url, request) == 'http://testserver/another/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + ) +) +def test_siteabsoluteurl_without_secured_proxy_adds_scheme_and_netloc(rf, log_warning_mock): + netloc_relative_url = '/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(netloc_relative_url, request) == 'http://testserver/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + ) +) +def test_siteabsoluteurl_without_secured_proxy_adds_scheme(rf, log_warning_mock): + scheme_relative_url = '//example.com/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(scheme_relative_url, request) == 'http://example.com/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + ) +) +def test_siteabsoluteurl_without_secured_proxy_returns_absolute_http_urls_unchanged(rf, log_warning_mock): + absolute_url = 'http://example.com/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(absolute_url, request) == 'http://example.com/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + ) +) +def test_siteabsoluteurl_without_secured_proxy_returns_absolute_https_urls_unchanged(rf, log_warning_mock): + absolute_url = 'https://example.com/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(absolute_url, request) == 'https://example.com/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + ) +) +def test_siteabsoluteurl_without_secured_proxy_returns_absolute_nonhttp_urls_unchanged(rf, log_warning_mock): + absolute_url = 'ftp://example.com/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(absolute_url, request) == 'ftp://example.com/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + SECURED_PROXY=True, + ) +) +def test_siteabsoluteurl_when_secured_proxy_in_use_adds_https_and_netloc_and_path_prefix(rf, log_warning_mock): + relative_url = 'foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(relative_url, request) == 'https://testserver/another/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + SECURED_PROXY=True, + ) +) +def test_siteabsoluteurl_when_secured_proxy_in_use_adds_https_and_netloc(rf, log_warning_mock): + netloc_relative_url = '/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(netloc_relative_url, request) == 'https://testserver/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + SECURED_PROXY=True, + ) +) +def test_siteabsoluteurl_when_secured_proxy_in_use_adds_https(rf, log_warning_mock): + scheme_relative_url = '//example.com/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(scheme_relative_url, request) == 'https://example.com/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + SECURED_PROXY=True, + ) +) +def test_siteabsoluteurl_when_secured_proxy_in_use_returns_absolute_http_urls_converted_to_https(rf, log_warning_mock): + absolute_url = 'http://example.com/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(absolute_url, request) == 'https://example.com/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + SECURED_PROXY=True, + ) +) +def test_siteabsoluteurl_when_secured_proxy_in_use_returns_absolute_https_urls_unchanged(rf, log_warning_mock): + absolute_url = 'https://example.com/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(absolute_url, request) == 'https://example.com/foo/bar' + log_warning_mock.assert_not_called() + + +@override_settings( + OSMAXX=dict( + SECURED_PROXY=True, + ) +) +def test_siteabsoluteurl_when_secured_proxy_in_use_returns_absolute_nonhttp_urls_unchanged(rf, log_warning_mock): + absolute_url = 'ftp://example.com/foo/bar' + request = rf.get('/another/path') + assert siteabsoluteurl(absolute_url, request) == 'ftp://example.com/foo/bar' + log_warning_mock.assert_called_with( + "ftp://example.com/foo/bar has not been converted to HTTPS, because it isn't an HTTP URL.") + + +@pytest.yield_fixture +def log_warning_mock(mocker): + original_level = logger.level + logger.setLevel(logging.WARNING) + yield mocker.patch.object(logger, 'warning') + logger.setLevel(original_level) diff --git a/tests/excerptexport/conftest.py b/tests/excerptexport/conftest.py index e3c96a606..188cca89e 100644 --- a/tests/excerptexport/conftest.py +++ b/tests/excerptexport/conftest.py @@ -1,9 +1,10 @@ import os import tempfile +from collections import namedtuple import pytest -from osmaxx.conversion_api import formats +from osmaxx.conversion import output_format from osmaxx.excerptexport.models import Excerpt, ExtractionOrder, Export, OutputFile @@ -28,7 +29,7 @@ def extraction_order(excerpt, user, db): return extraction_order -@pytest.fixture(params=[formats.FGDB, formats.SHAPEFILE, formats.GPKG, formats.SPATIALITE, formats.GARMIN]) +@pytest.fixture(params=output_format.ALL) def export(request, extraction_order): return Export.objects.create( extraction_order=extraction_order, @@ -36,6 +37,12 @@ def export(request, extraction_order): ) +@pytest.fixture +def exports(extraction_order): + ParamFake = namedtuple('ParamFake', 'param') + return [export(ParamFake(format), extraction_order) for format in output_format.ALL] + + @pytest.fixture def output_file(export): return OutputFile.objects.create(mime_type='test/plain', export=export) diff --git a/tests/excerptexport/models/test_export_model.py b/tests/excerptexport/models/test_export_model.py index 6d7a37362..716a1a234 100644 --- a/tests/excerptexport/models/test_export_model.py +++ b/tests/excerptexport/models/test_export_model.py @@ -23,10 +23,10 @@ def timestamp_model_mixin_save_side_effect(self, *args, **kwargs): @pytest.mark.django_db def test_created_export_has_correct_time_stamp(extraction_order): from osmaxx.excerptexport.models import Export - from osmaxx.conversion_api import formats + from osmaxx.conversion import output_format export = Export.objects.create( extraction_order=extraction_order, - file_format=formats.FGDB, + file_format=output_format.FGDB, ) now = timezone.now() margin = timedelta(minutes=1) diff --git a/tests/excerptexport/models/test_plain_text_email_rendering.py b/tests/excerptexport/models/test_plain_text_email_rendering.py new file mode 100644 index 000000000..9125c2829 --- /dev/null +++ b/tests/excerptexport/models/test_plain_text_email_rendering.py @@ -0,0 +1,19 @@ +from django.contrib.auth import get_user_model +from hamcrest import contains_string, match_equality + +from osmaxx.excerptexport.models import ExtractionOrder, Excerpt +from osmaxx.utils.shortcuts import Emissary + + +def test_send_email_if_all_exports_done_keeps_special_characters_unescaped(mocker, rf): + User = get_user_model() # noqa + extraction_order = ExtractionOrder(orderer=User(), excerpt=Excerpt(name='foo & bar')) + incoming_request = rf.get('') + im = mocker.patch.object(Emissary, 'inform_mail') + + extraction_order.send_email_if_all_exports_done(incoming_request) + + im.assert_called_once_with( + subject=match_equality(contains_string('foo & bar')), + mail_body=match_equality(contains_string('foo & bar')), + ) diff --git a/tests/excerptexport/permission_test_helper.py b/tests/excerptexport/permission_test_helper.py index 8e822560c..c3cc0069e 100644 --- a/tests/excerptexport/permission_test_helper.py +++ b/tests/excerptexport/permission_test_helper.py @@ -1,16 +1,10 @@ import pytest -from django.conf import settings -from django.contrib.auth.models import Group from osmaxx.profile.models import Profile @pytest.mark.django_db class PermissionHelperMixin(object): - def add_permissions_to_user(self): - group = Group.objects.get(name=settings.OSMAXX_FRONTEND_USER_GROUP) - self.user.groups.add(group) - def add_email(self): self.user.email = 'test@example.com' self.user.save() diff --git a/tests/excerptexport/templates/__init__.py b/tests/excerptexport/templates/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/excerptexport/templates/email/__init__.py b/tests/excerptexport/templates/email/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/excerptexport/templates/email/test_all_exports_of_extraction_order_done_subject.py b/tests/excerptexport/templates/email/test_all_exports_of_extraction_order_done_subject.py new file mode 100644 index 000000000..bb6e40348 --- /dev/null +++ b/tests/excerptexport/templates/email/test_all_exports_of_extraction_order_done_subject.py @@ -0,0 +1,136 @@ +from django.template.loader import render_to_string + + +def test_some_success_some_failed(rf, extraction_order, exports): + successful_exports = exports[::2] + failed_exports = exports[1::2] + + view_context = dict( + extraction_order=extraction_order, + successful_exports=successful_exports, + failed_exports=failed_exports, + request=rf.get('/foo/bar'), + ) + email_body = render_to_string( + 'excerptexport/email/all_exports_of_extraction_order_done_body.txt', + context=view_context, + ).strip() + expected_body = '\n'.join( + [ + 'This is an automated email from testserver', + '', + 'The extraction order #{order_id} "Neverland" has been processed and is available for download:', + '- Esri File Geodatabase', + '- GeoPackage', + '- Garmin navigation & map data', + '', + 'Unfortunately, the following exports have failed:', + '- Esri Shapefile', + '- SpatiaLite', + '- OSM Protocolbuffer Binary Format', + '', + 'Please order them anew if you need them. ' + 'If there are repeated failures, ' + 'please report them on https://github.com/geometalab/osmaxx/issues ' + 'unless the problem is already known there.', + '', + 'View the complete order at http://testserver/exports/ (login required)', + '', + 'Thank you for using OSMaxx.', + 'The team at Geometa Lab HSR', + 'geometalab@hsr.ch', + ] + ).format( + order_id=extraction_order.id, + ) + assert email_body == expected_body + + +def test_some_success_1_failed(rf, extraction_order, exports): + successful_exports = exports[:-1] + failed_exports = exports[-1:] + + view_context = dict( + extraction_order=extraction_order, + successful_exports=successful_exports, + failed_exports=failed_exports, + request=rf.get('/foo/bar'), + ) + email_body = render_to_string( + 'excerptexport/email/all_exports_of_extraction_order_done_body.txt', + context=view_context, + ).strip() + expected_body = '\n'.join( + [ + 'This is an automated email from testserver', + '', + 'The extraction order #{order_id} "Neverland" has been processed and is available for download:', + '- Esri File Geodatabase', + '- Esri Shapefile', + '- GeoPackage', + '- SpatiaLite', + '- Garmin navigation & map data', + '', + 'Unfortunately, the following export has failed:', + '- OSM Protocolbuffer Binary Format', + '', + 'Please order it anew if you need it. ' + 'If there are repeated failures, ' + 'please report them on https://github.com/geometalab/osmaxx/issues ' + 'unless the problem is already known there.', + '', + 'View the complete order at http://testserver/exports/ (login required)', + '', + 'Thank you for using OSMaxx.', + 'The team at Geometa Lab HSR', + 'geometalab@hsr.ch', + ] + ).format( + order_id=extraction_order.id, + ) + assert email_body == expected_body + + +def test_no_success_1_failed(rf, extraction_order, exports): + successful_exports = tuple() + failed_exports = exports + + view_context = dict( + extraction_order=extraction_order, + successful_exports=successful_exports, + failed_exports=failed_exports, + request=rf.get('/foo/bar'), + ) + email_body = render_to_string( + 'excerptexport/email/all_exports_of_extraction_order_done_body.txt', + context=view_context, + ).strip() + expected_body = '\n'.join( + [ + 'This is an automated email from testserver', + '', + 'The extraction order #{order_id} "Neverland" has been processed.', + '', + 'Unfortunately, the following exports have failed:', + '- Esri File Geodatabase', + '- Esri Shapefile', + '- GeoPackage', + '- SpatiaLite', + '- Garmin navigation & map data', + '- OSM Protocolbuffer Binary Format', + '', + 'Please order them anew if you need them. ' + 'If there are repeated failures, ' + 'please report them on https://github.com/geometalab/osmaxx/issues ' + 'unless the problem is already known there.', + '', + 'View the complete order at http://testserver/exports/ (login required)', + '', + 'Thank you for using OSMaxx.', + 'The team at Geometa Lab HSR', + 'geometalab@hsr.ch', + ] + ).format( + order_id=extraction_order.id, + ) + assert email_body == expected_body diff --git a/tests/excerptexport/test_conversion_api_client.py b/tests/excerptexport/test_conversion_api_client.py index 366dcb99a..c08002d90 100644 --- a/tests/excerptexport/test_conversion_api_client.py +++ b/tests/excerptexport/test_conversion_api_client.py @@ -8,9 +8,9 @@ from requests import HTTPError from rest_framework.reverse import reverse +from osmaxx.conversion import output_format from osmaxx.api_client import ConversionApiClient, API_client -from osmaxx.conversion_api.formats import FGDB, SPATIALITE -from osmaxx.conversion_api.statuses import RECEIVED +from osmaxx.conversion.constants.statuses import RECEIVED from osmaxx.excerptexport.models import Excerpt, ExtractionOrder from osmaxx.job_progress.views import tracker from tests.test_helpers import vcr_explicit_path as vcr @@ -76,7 +76,7 @@ def excerpt(user, bounding_geometry, db): @pytest.fixture def extraction_order(excerpt, user, db): extraction_order = ExtractionOrder.objects.create(excerpt=excerpt, orderer=user, id=23) - extraction_order.extraction_formats = [FGDB, SPATIALITE] + extraction_order.extraction_formats = [output_format.FGDB, output_format.SPATIALITE] extraction_order.coordinate_reference_system = 4326 return extraction_order @@ -104,8 +104,8 @@ def test_extraction_order_forward_to_conversion_service( detail_level = extraction_order.detail_level assert_that( ConversionApiClient.create_parametrization.mock_calls, contains_inanyorder( - mock.call(boundary=ConversionApiClient.create_boundary.return_value, out_format=FGDB, detail_level=detail_level, out_srs=srs), - mock.call(boundary=ConversionApiClient.create_boundary.return_value, out_format=SPATIALITE, detail_level=detail_level, out_srs=srs), + mock.call(boundary=ConversionApiClient.create_boundary.return_value, out_format=output_format.FGDB, detail_level=detail_level, out_srs=srs), + mock.call(boundary=ConversionApiClient.create_boundary.return_value, out_format=output_format.SPATIALITE, detail_level=detail_level, out_srs=srs), ) ) assert_that( @@ -114,8 +114,8 @@ def test_extraction_order_forward_to_conversion_service( mock.call(sentinel.parametrization_2, ANY, user=ANY), ) ) - fgdb_export = extraction_order.exports.get(file_format=FGDB) - spatialite_export = extraction_order.exports.get(file_format=SPATIALITE) + fgdb_export = extraction_order.exports.get(file_format=output_format.FGDB) + spatialite_export = extraction_order.exports.get(file_format=output_format.SPATIALITE) fgdb_callback_uri_path = reverse('job_progress:tracker', kwargs=dict(export_id=fgdb_export.id)) spatialite_callback_uri_path = reverse('job_progress:tracker', kwargs=dict(export_id=spatialite_export.id)) assert_that( @@ -132,8 +132,8 @@ def test_extraction_order_forward_to_conversion_service( ) assert_that( extraction_order.exports.values_list('file_format', flat=True), contains_inanyorder( - FGDB, - SPATIALITE, + output_format.FGDB, + output_format.SPATIALITE, ) ) assert_that( @@ -154,8 +154,8 @@ def api_client(): @vcr.use_cassette('fixtures/vcr/conversion_api-test_create_job.yml') def test_create_jobs_for_extraction_order(extraction_order, excerpt_request): - fgdb_export = extraction_order.exports.get(file_format=FGDB) - spatialite_export = extraction_order.exports.get(file_format=SPATIALITE) + fgdb_export = extraction_order.exports.get(file_format=output_format.FGDB) + spatialite_export = extraction_order.exports.get(file_format=output_format.SPATIALITE) assert fgdb_export.conversion_service_job_id is None assert fgdb_export.status == fgdb_export.INITIAL @@ -196,7 +196,7 @@ def clipping_area_json(): @vcr.use_cassette('fixtures/vcr/conversion_api-test_create_job_for_export.yml') def test_create_job_for_export(extraction_order, job_progress_request, clipping_area_json): - fgdb_export = extraction_order.exports.get(file_format=FGDB) + fgdb_export = extraction_order.exports.get(file_format=output_format.FGDB) job_json = fgdb_export.send_to_conversion_service(clipping_area_json, incoming_request=job_progress_request) assert job_json['callback_url'] == "http://the-host.example.com/job_progress/tracker/23/" assert job_json['rq_job_id'] == "6692fa44-cc19-4252-88ae-8687496da421" @@ -208,7 +208,7 @@ def test_create_job_for_export(extraction_order, job_progress_request, clipping_ @vcr.use_cassette('fixtures/vcr/conversion_api-test_create_job_for_export.yml') def test_callback_url_of_created_job_refers_to_correct_export(extraction_order, job_progress_request, clipping_area_json): - fgdb_export = extraction_order.exports.get(file_format=FGDB) + fgdb_export = extraction_order.exports.get(file_format=output_format.FGDB) job_json = fgdb_export.send_to_conversion_service(clipping_area_json, incoming_request=job_progress_request) @@ -222,7 +222,7 @@ def test_callback_url_of_created_job_refers_to_correct_export(extraction_order, @vcr.use_cassette('fixtures/vcr/conversion_api-test_create_job_for_export.yml') def test_callback_url_would_reach_this_django_instance(extraction_order, job_progress_request, the_host, clipping_area_json): - fgdb_export = extraction_order.exports.get(file_format=FGDB) + fgdb_export = extraction_order.exports.get(file_format=output_format.FGDB) job_json = fgdb_export.send_to_conversion_service(clipping_area_json, incoming_request=job_progress_request) diff --git a/tests/excerptexport/test_models.py b/tests/excerptexport/test_models.py index 11dc8d6f1..59c22f9da 100644 --- a/tests/excerptexport/test_models.py +++ b/tests/excerptexport/test_models.py @@ -3,8 +3,8 @@ from django.contrib.gis import geos from hamcrest import assert_that, contains_inanyorder as contains_in_any_order -from osmaxx.conversion_api.formats import GARMIN -from osmaxx.conversion_api.statuses import FINISHED +from osmaxx.conversion import output_format +from osmaxx.conversion.constants.statuses import FINISHED from osmaxx.excerptexport import models @@ -63,6 +63,6 @@ def test_retrieve_extraction_configuration_on_saved_extraction_configuration(sel def test_export_get_export_status_changed_message_does_not_html_escape_format_description(): excerpt = models.Excerpt(name='Obersee') extraction_order = models.ExtractionOrder(excerpt=excerpt) - export = models.Export(id=5, status=FINISHED, extraction_order=extraction_order, file_format=GARMIN) + export = models.Export(id=5, status=FINISHED, extraction_order=extraction_order, file_format=output_format.GARMIN) assert export._get_export_status_changed_message() == \ 'Export #5 "Obersee" to Garmin navigation & map data has finished.' diff --git a/tests/excerptexport/views/test_excerpt_delete_view.py b/tests/excerptexport/views/test_excerpt_delete_view.py index 9fdfbb7d4..5bbd78f32 100644 --- a/tests/excerptexport/views/test_excerpt_delete_view.py +++ b/tests/excerptexport/views/test_excerpt_delete_view.py @@ -4,7 +4,7 @@ from django.contrib.auth import get_user_model from django.views import generic -from osmaxx.conversion_api import formats +from osmaxx.conversion import output_format from osmaxx.excerptexport.models import Excerpt from osmaxx.excerptexport.models import Export from osmaxx.excerptexport.models import ExtractionOrder @@ -28,7 +28,7 @@ def test_delete_fails_when_exports_in_progress(authorized_client, user, extracti extraction_order.excerpt = excerpt extraction_order.save() Export.objects.create( - extraction_order=extraction_order, file_format=formats.FGDB, conversion_service_job_id=None, status=Export.INITIAL, finished_at=None + extraction_order=extraction_order, file_format=output_format.FGDB, conversion_service_job_id=None, status=Export.INITIAL, finished_at=None ) assert excerpt.has_running_exports diff --git a/tests/excerptexport/views/test_export_list.py b/tests/excerptexport/views/test_export_list.py index f5fac95f6..78a96d764 100644 --- a/tests/excerptexport/views/test_export_list.py +++ b/tests/excerptexport/views/test_export_list.py @@ -4,19 +4,20 @@ from django.contrib.gis import geos from django.core.urlresolvers import reverse from django.test import TestCase +from django.test.utils import override_settings from osmaxx.excerptexport.models import Excerpt, ExtractionOrder from tests.excerptexport.permission_test_helper import PermissionHelperMixin # TODO: test more possibilities +@override_settings(LOGIN_URL='/login/') @patch('osmaxx.job_progress.middleware.update_export') class ExportListTestCase(TestCase, PermissionHelperMixin): extraction_order = None def setUp(self): self.user = user = User.objects.create_user('user', 'user@example.com', 'pw') - self.client.login(username='user', password='pw') # FIXME: use the bounding_geometry fixture for this bounding_geometry = geos.GEOSGeometry( @@ -41,32 +42,20 @@ def setUp(self): orderer=User.objects.create_user('another_user', 'someone_else@example.com', 'top secret') ) - def test_permission_denied_if_lacking_permissions(self, *args): + def test_redirect_to_login_if_not_logged_in(self, *args): response = self.client.get( reverse( 'excerptexport:export_list' ) ) - # redirect to 'Access Denied' page - self.assertEqual(response.status_code, 302) + self.assertRedirects(response, '/login/?next=/exports/', fetch_redirect_response=False) - def test_permission_denied_if_lacking_email(self, *args): - self.add_permissions_to_user() - response = self.client.get( - reverse( - 'excerptexport:export_list' - ) - ) - # redirect to 'Access Denied' page - self.assertEqual(response.status_code, 302) + def test_access_ok_even_without_confirmed_email(self, *args): + self.client.login(username='user', password='pw') - def test_access_ok_if_permissions(self, *args): - self.add_permissions_to_user() - self.add_valid_email() response = self.client.get( reverse( 'excerptexport:export_list' ) ) - # redirect to 'Access Denied' page self.assertEqual(response.status_code, 200) diff --git a/tests/excerptexport/views/test_order_excerpt_views.py b/tests/excerptexport/views/test_order_excerpt_views.py index 97079a846..418280456 100644 --- a/tests/excerptexport/views/test_order_excerpt_views.py +++ b/tests/excerptexport/views/test_order_excerpt_views.py @@ -1,9 +1,9 @@ -from django.core.urlresolvers import reverse from django.contrib.auth.models import User +from django.core.urlresolvers import reverse from django.test import TestCase from hamcrest import assert_that, contains_inanyorder as contains_in_any_order -from osmaxx.conversion.converters.detail_levels import DETAIL_LEVEL_ALL +from osmaxx.conversion.converters.converter_gis.detail_levels import DETAIL_LEVEL_ALL from osmaxx.excerptexport.models import ExtractionOrder, Excerpt from tests.excerptexport.permission_test_helper import PermissionHelperMixin from tests.test_helpers import vcr_explicit_path as vcr @@ -67,7 +67,6 @@ def test_new(self): """ When logged in, we get the excerpt choice form. """ - self.add_permissions_to_user() self.add_valid_email() self.client.login(username='user', password='pw') response = self.client.get(reverse('excerptexport:order_new_excerpt')) @@ -76,7 +75,6 @@ def test_new(self): @vcr.use_cassette('fixtures/vcr/views-test_test_new_offers_existing_own_excerpt.yml') def test_new_offers_existing_own_excerpt(self): - self.add_permissions_to_user() self.add_valid_email() self.client.login(username='user', password='pw') response = self.client.get(reverse('excerptexport:order_existing_excerpt')) @@ -88,7 +86,6 @@ def test_new_offers_existing_own_excerpt(self): @vcr.use_cassette('fixtures/vcr/views-test_test_new_offers_existing_public_foreign_excerpt.yml') def test_new_offers_existing_public_foreign_excerpt(self): - self.add_permissions_to_user() self.add_valid_email() self.client.login(username='user', password='pw') response = self.client.get(reverse('excerptexport:order_existing_excerpt')) @@ -101,7 +98,6 @@ def test_new_offers_existing_public_foreign_excerpt(self): @vcr.use_cassette('fixtures/vcr/views-test_new_doesnt_offer_existing_private_foreign_excerpt.yml') def test_new_doesnt_offer_existing_private_foreign_excerpt(self): - self.add_permissions_to_user() self.add_valid_email() self.client.login(username='user', password='pw') response = self.client.get(reverse('excerptexport:order_existing_excerpt')) @@ -119,7 +115,6 @@ def test_create_with_new_excerpt(self): """ When logged in, POSTing an export request with a new excerpt is successful. """ - self.add_permissions_to_user() self.add_valid_email() self.client.login(username='user', password='pw') response = self.client.post( @@ -138,7 +133,6 @@ def test_create_with_new_excerpt_ignores_ispublic(self): """ When logged in, POSTing an export request with a new excerpt is successful. """ - self.add_permissions_to_user() self.add_valid_email() self.client.login(username='user', password='pw') self.new_excerpt_post_data['is_public'] = True @@ -162,7 +156,6 @@ def test_create_with_existing_excerpt(self): """ When logged in, POSTing an export request using an existing excerpt is successful. """ - self.add_permissions_to_user() self.add_valid_email() self.client.login(username='user', password='pw') response = self.client.post( @@ -180,7 +173,6 @@ def test_create_with_new_excerpt_persists_a_new_order(self): """ When logged in, POSTing an export request with a new excerpt persists a new ExtractionOrder. """ - self.add_permissions_to_user() self.add_valid_email() self.assertEqual(ExtractionOrder.objects.count(), 0) self.client.login(username='user', password='pw') @@ -200,7 +192,6 @@ def test_create_with_existing_excerpt_persists_a_new_order(self): """ When logged in, POSTing an export request using an existing excerpt persists a new ExtractionOrder. """ - self.add_permissions_to_user() self.add_valid_email() self.assertEqual(ExtractionOrder.objects.count(), 0) self.client.login(username='user', password='pw') diff --git a/tests/excerptexport/views/test_views_with_dowload_links.py b/tests/excerptexport/views/test_views_with_dowload_links.py index b02894f0d..0d2b982e8 100644 --- a/tests/excerptexport/views/test_views_with_dowload_links.py +++ b/tests/excerptexport/views/test_views_with_dowload_links.py @@ -5,7 +5,7 @@ from django.core.urlresolvers import reverse from django.test.testcases import SimpleTestCase -from osmaxx.conversion_api import formats +from osmaxx.conversion import output_format from osmaxx.excerptexport.models.excerpt import Excerpt from osmaxx.excerptexport.models.extraction_order import ExtractionOrder from osmaxx.excerptexport.models.output_file import OutputFile @@ -30,7 +30,7 @@ def order(excerpt, db): @pytest.fixture def exports(order, db): - return [order.exports.create(file_format=format_choice[0]) for format_choice in formats.FORMAT_CHOICES] + return [order.exports.create(file_format=format_choice[0]) for format_choice in output_format.CHOICES] @pytest.fixture diff --git a/tests/job_progress/test_job_progress.py b/tests/job_progress/test_job_progress.py index 71f6fc561..387ac50de 100644 --- a/tests/job_progress/test_job_progress.py +++ b/tests/job_progress/test_job_progress.py @@ -13,7 +13,7 @@ from osmaxx import excerptexport from osmaxx.api_client.conversion_api_client import ConversionApiClient -from osmaxx.conversion_api.statuses import STARTED, QUEUED, FINISHED, FAILED +from osmaxx.conversion.constants.statuses import STARTED, QUEUED, FINISHED, FAILED from osmaxx.excerptexport.models.excerpt import Excerpt from osmaxx.excerptexport.models.export import Export from osmaxx.excerptexport.models.extraction_order import ExtractionOrder @@ -81,7 +81,7 @@ def test_calling_tracker_with_payload_indicating_started_updates_export_status(s self.export.refresh_from_db() self.assertEqual(self.export.status, STARTED) - @patch('osmaxx.utilities.shortcuts.Emissary') + @patch('osmaxx.utils.shortcuts.Emissary') def test_calling_tracker_with_payload_indicating_queued_informs_user( self, emissary_class_mock, *args, **mocks): emissary_mock = emissary_class_mock() @@ -103,7 +103,7 @@ def test_calling_tracker_with_payload_indicating_queued_informs_user( ) ) - @patch('osmaxx.utilities.shortcuts.Emissary') + @patch('osmaxx.utils.shortcuts.Emissary') def test_calling_tracker_with_payload_indicating_started_informs_user( self, emissary_class_mock, *args, **mocks): emissary_mock = emissary_class_mock() @@ -118,14 +118,15 @@ def test_calling_tracker_with_payload_indicating_started_informs_user( assert_that( emissary_mock.mock_calls, contains_in_any_order( call.info( - 'Export #{export_id} "Neverland" to Esri File Geodatabase has been started.'.format( + 'Export #{export_id} "Neverland" to Esri File Geodatabase' + ' has been started. Exporting will take around 30 minutes.'.format( export_id=self.export.id ), ), ) ) - @patch('osmaxx.utilities.shortcuts.Emissary') + @patch('osmaxx.utils.shortcuts.Emissary') def test_calling_tracker_with_payload_indicating_failed_informs_user_with_error( self, emissary_class_mock, *args, **mocks): emissary_mock = emissary_class_mock() @@ -148,7 +149,7 @@ def test_calling_tracker_with_payload_indicating_failed_informs_user_with_error( ) @patch.object(Export, '_fetch_result_file') - @patch('osmaxx.utilities.shortcuts.Emissary') + @patch('osmaxx.utils.shortcuts.Emissary') def test_calling_tracker_with_payload_indicating_finished_informs_user_with_success( self, emissary_class_mock, *args, **mocks): emissary_mock = emissary_class_mock() @@ -170,7 +171,7 @@ def test_calling_tracker_with_payload_indicating_finished_informs_user_with_succ ) ) - @patch('osmaxx.utilities.shortcuts.Emissary') + @patch('osmaxx.utils.shortcuts.Emissary') def test_calling_tracker_with_payload_indicating_unchanged_status_does_not_inform_user( self, emissary_class_mock, *args, **mocks): self.export.status = 'started' @@ -221,7 +222,7 @@ def test_calling_tracker_with_payload_indicating_status_finished_downloads_resul @requests_mock.Mocker(kw='requests') @patch.object(ConversionApiClient, 'authorized_get', ConversionApiClient.get) # circumvent authorization logic - @patch('osmaxx.utilities.shortcuts.Emissary') + @patch('osmaxx.utils.shortcuts.Emissary') def test_calling_tracker_with_payload_indicating_final_status_for_only_remaining_nonfinal_export_of_extraction_order_advertises_downloads( self, emissary_class_mock, *args, **mocks): self.export.conversion_service_job_id = 1 @@ -263,9 +264,10 @@ def test_calling_tracker_with_payload_indicating_final_status_for_only_remaining 'The extraction order #{order_id} "Neverland" has been processed and is available for download:', '- Esri File Geodatabase: http://testserver{download_url}', '', - 'Unfortunately, the following exports have failed:', + 'Unfortunately, the following export has failed:', '- SpatiaLite', - 'Please order them anew if you need them. ' + '', + 'Please order it anew if you need it. ' 'If there are repeated failures, ' 'please report them on https://github.com/geometalab/osmaxx/issues ' 'unless the problem is already known there.', diff --git a/tests/profile/test_profile_view.py b/tests/profile/test_profile_view.py index a0a01ab5d..6d5b4c236 100644 --- a/tests/profile/test_profile_view.py +++ b/tests/profile/test_profile_view.py @@ -78,7 +78,7 @@ def test_email_change(authorized_client): @pytest.mark.django_db() def test_mail_sent_only_once_within_rate_limit(authenticated_client): - with patch('osmaxx.profile.views.send_mail') as send_mail: + with patch('osmaxx.profile.email_confirmation.send_mail') as send_mail: assert send_mail.call_count == 0 authenticated_client.get(reverse('profile:edit_view')) assert send_mail.call_count == 1 diff --git a/tests/profile/test_resend_verfication_email_view.py b/tests/profile/test_resend_verfication_email_view.py index 4abd8c910..ecb712dd9 100644 --- a/tests/profile/test_resend_verfication_email_view.py +++ b/tests/profile/test_resend_verfication_email_view.py @@ -10,7 +10,7 @@ def test_resend_verification_page_redirects_when_called_without_user(client): def test_resend_verification_page_sends_mail_with_profile(authorized_client, valid_profile): - with patch('osmaxx.profile.views.send_mail') as send_mail: + with patch('osmaxx.profile.email_confirmation.send_mail') as send_mail: assert send_mail.call_count == 0 response = authorized_client.get(reverse('profile:resend_verification'), follow=True) assert response.status_code == 200 @@ -18,8 +18,8 @@ def test_resend_verification_page_sends_mail_with_profile(authorized_client, val def test_resend_verification_page_limits_click_ratio(authorized_client, valid_profile): - with patch('osmaxx.profile.views.send_mail') as send_mail: - with patch('osmaxx.profile.views.cache') as cache: + with patch('osmaxx.profile.email_confirmation.send_mail') as send_mail: + with patch('osmaxx.profile.email_confirmation.cache') as cache: cache.get.return_value = None assert send_mail.call_count == 0 authorized_client.get(reverse('profile:resend_verification'), follow=True) diff --git a/tests/utilities/test_models/migrations/0001_initial.py b/tests/utilities/test_models/migrations/0001_initial.py deleted file mode 100644 index 855285dda..000000000 --- a/tests/utilities/test_models/migrations/0001_initial.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Article', - fields=[ - ('id', models.AutoField(verbose_name='ID', auto_created=True, serialize=False, primary_key=True)), - ('title', models.CharField(max_length=50)), - ], - ), - migrations.CreateModel( - name='Author', - fields=[ - ('id', models.AutoField(verbose_name='ID', auto_created=True, serialize=False, primary_key=True)), - ('name', models.CharField(max_length=50)), - ], - ), - migrations.AddField( - model_name='article', - name='authors', - field=models.ManyToManyField(to='test_models.Author'), - ), - ] diff --git a/tests/utilities/test_models/models.py b/tests/utilities/test_models/models.py deleted file mode 100644 index 253e7a9c2..000000000 --- a/tests/utilities/test_models/models.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -DB-API Shortcuts -``get_object_or_none()`` is a shortcut function to be used in view functions for -performing a ``get()`` lookup and return ``None`` if a -``DoesNotExist`` exception was raised during the ``get()`` call. -``get_list_or_none()`` is a shortcut function to be used in view functions for -performing a ``filter()`` lookup and returning ``None`` if a -``DoesNotExist`` exception was raised during the ``filter()`` call. -""" - -from django.db import models - - -class Author(models.Model): - name = models.CharField(max_length=50) - - def __str__(self): - return self.name - - -class ArticleManager(models.Manager): - def get_queryset(self): - return super(ArticleManager, self).get_queryset().filter(authors__name__icontains='sir') - - -class Article(models.Model): - authors = models.ManyToManyField(Author) - title = models.CharField(max_length=50) - objects = models.Manager() - by_a_sir = ArticleManager() - - def __str__(self): - return self.title diff --git a/tests/utilities/test_dict_helpers.py b/tests/utils/test_dict_helpers.py similarity index 92% rename from tests/utilities/test_dict_helpers.py rename to tests/utils/test_dict_helpers.py index b48fc32f8..c18df6954 100644 --- a/tests/utilities/test_dict_helpers.py +++ b/tests/utils/test_dict_helpers.py @@ -1,6 +1,6 @@ from unittest.case import TestCase -from osmaxx.utilities.dict_helpers import are_all_keys_in +from osmaxx.utils.dict_helpers import are_all_keys_in class AreAllKeysInTest(TestCase): diff --git a/tests/utils/test_logged_subprocess_wrapper.py b/tests/utils/test_logged_subprocess_wrapper.py new file mode 100644 index 000000000..cbdbe6526 --- /dev/null +++ b/tests/utils/test_logged_subprocess_wrapper.py @@ -0,0 +1,44 @@ +import pytest +import subprocess + +from osmaxx.conversion.converters import utils + + +@pytest.fixture +def subprocess_check_call_mock(mocker): + return mocker.patch.object(subprocess, 'check_call') + + +def test_logged_check_call_forwards_args(subprocess_check_call_mock): + test_call = ['echo', '0'] + utils.logged_check_call(test_call) + subprocess_check_call_mock.assert_called_with(test_call) + + +def test_logged_check_call_forwards_kwargs(subprocess_check_call_mock): + test_call = "echo 0" + utils.logged_check_call(test_call, shell=True) + subprocess_check_call_mock.assert_called_with(test_call, shell=True) + + +def test_logged_check_call_raises(): + with pytest.raises(subprocess.CalledProcessError) as excinfo: + utils.logged_check_call(['false']) + assert excinfo.value.output is None + assert excinfo.value.returncode == 1 + + +def test_logged_check_call_logs_error_content(mocker, subprocess_check_call_mock): + error_output = 'example output' + error_return_code = 17 + command = 'cmd' + + error = subprocess.CalledProcessError(cmd=command, output=error_output, returncode=error_return_code) + subprocess_check_call_mock.side_effect = error + logger_mock = mocker.patch.object(utils.logger, 'error') + + with pytest.raises(subprocess.CalledProcessError): + utils.logged_check_call(command) + + expected_output = 'Command `cmd` exited with return value 17\nOutput:\nexample output' + logger_mock.assert_called_with(expected_output) diff --git a/web_frontend/config/settings/common.py b/web_frontend/config/settings/common.py index d7a58f3b7..d1af39f12 100644 --- a/web_frontend/config/settings/common.py +++ b/web_frontend/config/settings/common.py @@ -56,7 +56,6 @@ LOCAL_APPS = ( 'osmaxx.version', - 'osmaxx.conversion_api', 'osmaxx.excerptexport', 'osmaxx.job_progress', 'osmaxx.profile', @@ -160,6 +159,13 @@ # See: https://docs.djangoproject.com/en/dev/ref/settings/#use-tz USE_TZ = True +# See https://docs.djangoproject.com/en/dev/ref/settings/#datetime-format +DATETIME_FORMAT = "Y-m-d H:i:s e" # e.g. '2017-03-21 16:15:20 CET' + +# See https://mounirmesselmeni.github.io/2014/11/06/date-format-in-django-admin/ +from django.conf.locale.en import formats as en_formats +en_formats.DATETIME_FORMAT = DATETIME_FORMAT + # TEMPLATE CONFIGURATION # ------------------------------------------------------------------------------ # See: https://docs.djangoproject.com/en/1.8/ref/templates/upgrading/#the-templates-settings @@ -367,7 +373,6 @@ # Do not alter this once migrations have been run, since these values are stored in the database. OSMAXX_FRONTEND_USER_GROUP = 'osmaxx_frontend_users' -REGISTRATION_OPEN = True # If True, verified users are automatically being added to OSMAXX_FRONTEND_USER_GROUP # message type mapping MESSAGE_TAGS = { @@ -393,8 +398,6 @@ 'OLD_RESULT_FILES_REMOVAL_CHECK_INTERVAL': timezone.timedelta( hours=env.int('DJANGO_OSMAXX_OLD_RESULT_FILES_REMOVAL_CHECK_INTERVAL_HOURS', default=1) ), - # The email adress of this user will be used to generate the mailto link for users - # to request access to osmaxx (access_denied page) 'ACCOUNT_MANAGER_EMAIL': env.str('OSMAXX_ACCOUNT_MANAGER_EMAIL', default=DEFAULT_FROM_EMAIL), 'CONVERSION_SERVICE_URL': env.str('DJANGO_OSMAXX_CONVERSION_SERVICE_URL', default='http://localhost:8901/api/'), 'CONVERSION_SERVICE_USERNAME': env.str('DJANGO_OSMAXX_CONVERSION_SERVICE_USERNAME', default='default_user'),