Merge branch 'develop' into feature/#167_pbf_export_format

.gitignore

@@ -50,7 +50,7 @@ nosetests.xml
 coverage.xml
 
 # Translations
-*.mo
+# *.mo  # (We DO track these, as we don't want to compile translations for every installation.)
 *.pot
 
 # Django stuff:

Dockerfile.mediator

@@ -24,11 +24,11 @@ RUN pip3 install -r requirements.txt
 
 # TODO: this is just a temporary solution, use pip for production as soon as geometalab.osmaxx is published there
 ADD ./osmaxx $HOME/osmaxx
-ADD ./osmaxx_conversion_service $HOME/osmaxx_conversion_service
+ADD ./conversion_service $HOME/conversion_service
 
 # Expose modules:
 ENV PYTHONPATH=PYTHONPATH:$HOME
-ENV DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.production
+ENV DJANGO_SETTINGS_MODULE=conversion_service.config.settings.production
 
 RUN mkdir -p $HOME/docker_entrypoint/osmaxx/conversion_service $HOME/entrypoint
 COPY ./docker_entrypoint/osmaxx/conversion_service $HOME/entrypoint

Dockerfile.worker

@@ -104,10 +104,12 @@ RUN apt-get clean && apt-get update && \
 WORKDIR /root/osm2pgsql
 
 # OSM2PGSQL
+ENV OSM2PGSQL_VERSION=0.92.0
 RUN mkdir src &&\
     cd src &&\
     GIT_SSL_NO_VERIFY=true git clone https://github.com/openstreetmap/osm2pgsql.git &&\
     cd osm2pgsql &&\
+    git checkout ${OSM2PGSQL_VERSION} &&\
     mkdir build &&\
     cd build &&\
     cmake ..&&\
@@ -166,11 +168,11 @@ RUN pip3 install -r requirements.txt
 
 # TODO: this is just a temporary solution, use pip for production as soon as geometalab.osmaxx is published there
 ADD ./osmaxx $HOME/osmaxx
-ADD ./osmaxx_conversion_service $HOME/osmaxx_conversion_service
+ADD ./conversion_service $HOME/conversion_service
 
 # expose modules
 ENV PYTHONPATH=PYTHONPATH:$HOME
-ENV DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.worker
+ENV DJANGO_SETTINGS_MODULE=conversion_service.config.settings.worker
 ENV WORKER_QUEUES default high
 
 ENTRYPOINT ["/home/py/entrypoint/entrypoint.sh"]

conversion_service/Procfile.mediator.dev

@@ -0,0 +1,2 @@
+mediator: python3 ./conversion_service/manage.py runserver_plus ${APP_HOST}:${APP_PORT}
+harvester: python3 ./conversion_service/manage.py result_harvester

conversion_service/Procfile.mediator.prod

@@ -0,0 +1,2 @@
+mediator: gunicorn --workers ${NUM_WORKERS} conversion_service.config.wsgi --bind ${APP_HOST}:${APP_PORT}
+harvester: python3 ./conversion_service/manage.py result_harvester

conversion_service/Procfile.worker

@@ -0,0 +1 @@
+worker: ${HOME}/entrypoint/wait-for-it.sh localhost:5432 -t 30 && python3 ./conversion_service/manage.py rqworker ${WORKER_QUEUES:-default high}

osmaxx_conversion_service/config/settings/common.py → conversion_service/config/settings/common.py

@@ -43,7 +43,6 @@
 LOCAL_APPS = (
     'osmaxx.version',
     'osmaxx.clipping_area',
-    'osmaxx.conversion_api',
     'osmaxx.conversion',
 )
 
@@ -190,10 +189,10 @@
 
 # URL Configuration
 # ------------------------------------------------------------------------------
-ROOT_URLCONF = 'osmaxx_conversion_service.config.urls'
+ROOT_URLCONF = 'conversion_service.config.urls'
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
-WSGI_APPLICATION = 'osmaxx_conversion_service.config.wsgi.application'
+WSGI_APPLICATION = 'conversion_service.config.wsgi.application'
 
 CACHES = {
     'default': {

osmaxx_conversion_service/config/wsgi.py → conversion_service/config/wsgi.py

@@ -12,7 +12,7 @@
 from whitenoise.django import DjangoWhiteNoise
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "osmaxx_conversion_service.config.settings.production")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conversion_service.config.settings.production")
 
 application = get_wsgi_application()
 application = Sentry(DjangoWhiteNoise(application))

osmaxx_conversion_service/manage.py → conversion_service/manage.py

@@ -5,7 +5,7 @@
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
 if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "osmaxx_conversion_service.config.settings.local")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conversion_service.config.settings.local")
 
     from django.core.management import execute_from_command_line
 

docker-compose-dev.yml

@@ -13,7 +13,7 @@ services:
     volumes:
       - ./osmaxx:/home/py/osmaxx
       - ./web_frontend:/home/py/web_frontend
-      - ./osmaxx_conversion_service:/home/py/osmaxx_conversion_service
+      - ./conversion_service:/home/py/conversion_service
     ports:
       - "8000:8000"
     command: [honcho, -f, ./web_frontend/Procfile.django.dev, start]
@@ -42,11 +42,11 @@ services:
     volumes:
       - ./osmaxx:/home/py/osmaxx
       - ./web_frontend:/home/py/web_frontend
-      - ./osmaxx_conversion_service:/home/py/osmaxx_conversion_service
-    command: [honcho, -f, ./osmaxx_conversion_service/Procfile.mediator.dev, start]
+      - ./conversion_service:/home/py/conversion_service
+    command: [honcho, -f, ./conversion_service/Procfile.mediator.dev, start]
     environment:
       - DJANGO_SECRET_KEY=insecure!2
-      - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.local
+      - DJANGO_SETTINGS_MODULE=conversion_service.config.settings.local
     logging:
       driver: "json-file"
       options:
@@ -55,7 +55,7 @@ services:
     volumes:
       - ./osmaxx:/home/py/osmaxx
       - ./web_frontend:/home/py/web_frontend
-      - ./osmaxx_conversion_service:/home/py/osmaxx_conversion_service
+      - ./conversion_service:/home/py/conversion_service
     build:
       context: .
       dockerfile: Dockerfile.worker
@@ -69,7 +69,7 @@ services:
     volumes:
       - ./osmaxx:/home/py/osmaxx
       - ./web_frontend:/home/py/web_frontend
-      - ./osmaxx_conversion_service:/home/py/osmaxx_conversion_service
+      - ./conversion_service:/home/py/conversion_service
     build:
       context: .
       dockerfile: Dockerfile.worker

docker-compose.yml

@@ -60,7 +60,7 @@ services:
     extends:
       file: docker-compose-common.yml
       service: conversionbase
-    command: [honcho, -f, ./osmaxx_conversion_service/Procfile.mediator.prod, start]
+    command: [honcho, -f, ./conversion_service/Procfile.mediator.prod, start]
     expose:
       - "8901"
     volumes:
@@ -77,7 +77,7 @@ services:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
       # comma separated list, no brackets, e.g. localhost,dev.myhost.com
-      - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.production
+      - DJANGO_SETTINGS_MODULE=conversion_service.config.settings.production
       - NUM_WORKERS=5
     depends_on:
       - conversionserviceredis
@@ -89,7 +89,7 @@ services:
     extends:
       file: docker-compose-common.yml
       service: conversionbase
-    command: [honcho, -f, ./osmaxx_conversion_service/Procfile.worker, start]
+    command: [honcho, -f, ./conversion_service/Procfile.worker, start]
     volumes:
       - osm_data:/var/data/osm-planet
       - worker-data:/data/media/job_result_files
@@ -101,7 +101,7 @@ services:
     environment:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
-      - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.worker
+      - DJANGO_SETTINGS_MODULE=conversion_service.config.settings.worker
     depends_on:
       - conversionserviceredis
       - osmboundaries-database
@@ -113,7 +113,7 @@ services:
     extends:
       file: docker-compose-common.yml
       service: conversionbase
-    command: [honcho, -f, ./osmaxx_conversion_service/Procfile.worker, start]
+    command: [honcho, -f, ./conversion_service/Procfile.worker, start]
     volumes:
       - osm_data:/var/data/osm-planet
       - worker-data:/data/media/job_result_files
@@ -125,7 +125,7 @@ services:
     environment:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
-      - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.worker
+      - DJANGO_SETTINGS_MODULE=conversion_service.config.settings.worker
       # only listens for jobs in this specific queue
       - WORKER_QUEUES=high
     depends_on:

docker_entrypoint/osmaxx/conversion_service/entrypoint.sh

@@ -4,7 +4,7 @@ set -e
 # wait for at most 30s for the db to be up
 ${HOME}/entrypoint/wait-for-it.sh ${DATABASE_HOST}:${DATABASE_PORT} -t 30
 
-python3 osmaxx_conversion_service/manage.py migrate --no-input && \
-python3 osmaxx_conversion_service/manage.py collectstatic --noinput && \
+python3 conversion_service/manage.py migrate --no-input && \
+python3 conversion_service/manage.py collectstatic --noinput && \
 python3 entrypoint/create_user_entrypoint.py && \
 exec "$@"

docs/development/releasing.md

@@ -22,7 +22,7 @@ The steps execute in the script in a nutshell are:
 - `git flow release publish <version>`
 - `adapt <version> in osmaxx/__init__.py`
 - `python ./web_frontend/manage.py makemessages -l de_CH -l en -l en_UK -l en_US`
-- `python ./osmaxx_conversion_service/manage.py makemessages -l de_CH -l en -l en_UK -l en_US`
+- `python ./conversion_service/manage.py makemessages -l de_CH -l en -l en_UK -l en_US`
 - `git flow release finish <version>`
 - `git push`
 - `git push --tags`

osmaxx/__init__.py

@@ -1,4 +1,4 @@
-__version__ = 'v3.5.0'
+__version__ = 'v3.10.0'
 
 __all__ = [
     '__version__',

osmaxx/contrib/auth/frontend_permissions.py

@@ -1,19 +1,11 @@
-from django.contrib.auth.decorators import login_required, user_passes_test
+from django.contrib.auth.decorators import user_passes_test
 from django.core.urlresolvers import reverse_lazy
 from django.utils.decorators import method_decorator
 from rest_framework import permissions
 
 from osmaxx.profile.models import Profile
 
 
-def _may_user_access_osmaxx_frontend(user):
-    """
-    Actual test to check if the user is in the frontend user group,
-    to give access or deny it. Note: Admins have superpowers.
-    """
-    return user.has_perm('excerptexport.add_extractionorder')
-
-
 def _may_user_access_this_excerpt(user, excerpt):
     return excerpt.is_public or excerpt.owner == user
 
@@ -30,21 +22,6 @@ def _user_has_validated_email(user):
     return profile.has_validated_email()
 
 
-def frontend_access_required(function=None):
-    """
-    Decorator for views that checks that the user has the correct access rights,
-    redirecting to the information page if necessary.
-    """
-    access_denied_info_url = reverse_lazy('excerptexport:access_denied')
-    actual_decorator = user_passes_test(
-        _may_user_access_osmaxx_frontend,
-        login_url=access_denied_info_url
-    )
-    if function:
-        return actual_decorator(function)
-    return actual_decorator
-
-
 def validated_email_required(function=None):
     """
     Decorator for views that checks that the user has set a validated email,
@@ -60,24 +37,6 @@ def validated_email_required(function=None):
     return actual_decorator
 
 
-class LoginRequiredMixin(object):
-    """
-    Login required Mixin for Class Based Views.
-    """
-    @method_decorator(login_required)
-    def dispatch(self, *args, **kwargs):
-        return super(LoginRequiredMixin, self).dispatch(*args, **kwargs)
-
-
-class FrontendAccessRequiredMixin(object):
-    """
-    Frontend Access Check Mixin for Class Based Views.
-    """
-    @method_decorator(frontend_access_required)
-    def dispatch(self, *args, **kwargs):
-        return super(FrontendAccessRequiredMixin, self).dispatch(*args, **kwargs)
-
-
 class EmailRequiredMixin(object):
     """
     Frontend Access Check Mixin for Class Based Views.
@@ -89,11 +48,11 @@ def dispatch(self, *args, **kwargs):
 
 class AuthenticatedAndAccessPermission(permissions.BasePermission):
     """
-    Allows access only to authenticated users with frontend permissions.
+    Allows access only to authenticated users with confirmed email address.
     """
 
     def has_permission(self, request, view):
-        return request.user.is_authenticated and _may_user_access_osmaxx_frontend(request.user)
+        return request.user.is_authenticated and _user_has_validated_email(request.user)
 
 
 class HasBBoxAccessPermission(permissions.BasePermission):

osmaxx/conversion/__init__.py

@@ -1 +1,5 @@
+from .constants import output_format
+
 default_app_config = 'osmaxx.conversion.apps.ConversionConfig'
+
+__all__ = ['default_app_config', 'output_format']

osmaxx/conversion_api/formats.py → osmaxx/conversion/constants/output_format.py

@@ -1,62 +1,45 @@
 import uuid
-from collections import OrderedDict
+from collections import OrderedDict, namedtuple
 
-from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy as _
 
 FGDB, SHAPEFILE, GPKG, SPATIALITE, GARMIN, PBF = 'fgdb', 'shapefile', 'gpkg', 'spatialite', 'garmin', 'pbf'
 
 
-class OutputFormat:
-    def __init__(
-            self, *, long_identifier, verbose_name, archive_file_name_identifier, abbreviations, is_white_box,
-            layer_filename_extension=None
-    ):
-        self._long_identifier = long_identifier
-        self._verbose_name = verbose_name
-        self._archive_file_name_identifier = archive_file_name_identifier
-        self._abbreviations = abbreviations
-        self._is_white_box = is_white_box
-        self._layer_filename_extension = layer_filename_extension
+_OutputFormatBase = namedtuple(
+    'OutputFormatBase',
+    [
+        'long_identifier',
+        'verbose_name',
+        'archive_file_name_identifier',
+        'abbreviations',
+        'is_white_box',
+        'layer_filename_extension',
+    ]
+)
+_OutputFormatBase.__new__.__defaults__ = (None,)  # This makes the last field (i.e. layer_filename_extension) optional.
 
-    @property
-    def long_identifier(self):
-        return self._long_identifier
-
-    @property
-    def verbose_name(self):
-        return self._verbose_name
-
-    @property
-    def archive_file_name_identifier(self):
-        return self._archive_file_name_identifier
-
-    @property
-    def abbreviations(self):
-        return self._abbreviations
-
-    @property
-    def layer_filename_extension(self):
-        return self._layer_filename_extension
 
+class OutputFormat(_OutputFormatBase):
     @property
     def qgis_datasource_separator(self):
         """
         The string used to separate the dataset path (collections of layers) from the individual layer in a QGIS project
         file's ``<datasource>`` element referring to data in this format.
         """
-        return '/' if self._layer_filename_extension is not None else '|layername='
+        return '/' if self.layer_filename_extension is not None else '|layername='
 
     def unique_archive_name(self):
         return "{}_{}.zip".format(uuid.uuid4(), self.archive_file_name_identifier)
 
     def crs_change_available(self):
-        return self._is_white_box
+        return self.is_white_box
 
     def detail_level_available(self):
-        return self._is_white_box
+        return self.is_white_box
 
 
-FORMAT_DEFINITIONS = OrderedDict([
+DEFINITIONS = OrderedDict([
     (FGDB, OutputFormat(
         long_identifier='Esri File Geodatabase',
         verbose_name=_('Esri File Geodatabase'),
@@ -102,4 +85,5 @@ def detail_level_available(self):
     )),
 ])
 
-FORMAT_CHOICES = tuple((key, definition.verbose_name) for key, definition in FORMAT_DEFINITIONS.items())
+CHOICES = tuple((key, definition.verbose_name) for key, definition in DEFINITIONS.items())
+ALL = DEFINITIONS.keys()