Merge branch 'release/v2.2.0'

.travis.yml

@@ -5,16 +5,11 @@ cache: pip
 sudo: required
 
 env:
-    - TOX_ENV=py34-flake8
-    - TOX_ENV=py34-django1.8-drf3.2
+    - TOX_ENV=py35-flake8
     - TOX_ENV=py34-django1.9-drf3.2
-    - TOX_ENV=py34-django1.8-drf3.3
     - TOX_ENV=py34-django1.9-drf3.3
     - TOX_ENV=py34-slow-tests
-    - TOX_ENV=py35-flake8
-    - TOX_ENV=py35-django1.8-drf3.2
     - TOX_ENV=py35-django1.9-drf3.2
-    - TOX_ENV=py35-django1.8-drf3.3
     - TOX_ENV=py35-django1.9-drf3.3
     - TOX_ENV=py35-slow-tests
 

Dockerfile.frontend

@@ -1,19 +1,16 @@
 FROM geometalab/python3-gis
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y git
+RUN apt-get clean && DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    git
 
 ENV USER py
 
 ENV HOME /home/$USER
 
 WORKDIR $HOME
 
-# if you update your requirements, please update this to the actual date/time, 
-# otherwise docker uses the cache from the intermediate image build (not re-running pip3).
-ENV REQS_LAST_UPDATED 04-05-2016 8:50
-
+RUN pip3 install honcho
 ADD ./requirements.txt $HOME/requirements.txt
-
 RUN pip3 install -r requirements.txt
 
 # TODO: this is just a temporary solution, use pip for production as soon as geometalab.osmaxx is published there

Dockerfile.mediator

@@ -4,23 +4,24 @@ USER root
 
 MAINTAINER HSR Geometalab <[email protected]>
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+RUN apt-get clean && DEBIAN_FRONTEND=noninteractive apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y\
 \
     libgeos-dev \
     libgeos++-dev \
-    python3-pip
+    python3-pip \
+    liblapack-dev \
+    gfortran
 
 # Install required Python packages:
 ENV HOME /home/py
 
 WORKDIR $HOME
 
-ENV REQS_LAST_UPDATED 04-05-2016 8:50
-ADD ./requirements.txt $HOME/
-
 RUN pip3 install honcho
-
+ADD ./requirements-mediator.txt $HOME/
+ADD ./requirements.txt $HOME/
+RUN pip3 install -r requirements-mediator.txt
 RUN pip3 install -r requirements.txt
 
 # TODO: this is just a temporary solution, use pip for production as soon as geometalab.osmaxx is published there

Dockerfile.nginx

@@ -1,2 +1,5 @@
 FROM nginx:alpine
-COPY ./docker_entrypoint/nginx/default.conf /etc/nginx/conf.d/default.conf
+COPY ./docker_entrypoint/nginx/default.conf.template /etc/nginx/conf.d/default.conf.template
+CMD DOMAIN_NAMES=$(echo $VIRTUAL_HOST | sed 's/,/ /g') envsubst '$DOMAIN_NAMES' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf \
+    && cat /etc/nginx/conf.d/default.conf \
+    && nginx -g 'daemon off;'

Dockerfile.worker

@@ -5,21 +5,25 @@ USER root
 
 # make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
 RUN apt-get update && apt-get install -y apt-utils locales && rm -rf /var/lib/apt/lists/* \
-    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ################## SETUP POSTGIS DATABASE with UTF8 support #############
 # explicitly set user/group IDs
 RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres
 
 # grab gosu for easy step-down from root
 RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \
+RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \
     && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/1.2/gosu-$(dpkg --print-architecture)" \
     && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/1.2/gosu-$(dpkg --print-architecture).asc" \
     && gpg --verify /usr/local/bin/gosu.asc \
     && rm /usr/local/bin/gosu.asc \
     && chmod +x /usr/local/bin/gosu \
-    && apt-get purge -y --auto-remove ca-certificates wget
+    && apt-get purge -y --auto-remove ca-certificates wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir /docker-entrypoint-initdb.d
 
@@ -40,6 +44,7 @@ RUN apt-get update \
         postgresql-${PG_MAJOR}-postgis-scripts \
         postgresql-server-dev-${PG_MAJOR} \
         postgresql-contrib-${PG_MAJOR} \
+    && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /var/run/postgresql && chown -R 999:999 /var/run/postgresql
@@ -53,7 +58,7 @@ RUN pg_createcluster --locale=en_US.UTF-8 -d $PGDATA 9.4 main
 
 ################## END SETUP POSTGIS DATABASE with UTF8 support #############
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+RUN apt-get clean && apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y\
 \
     make \
@@ -96,7 +101,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
     debhelper \
 \
     default-jre \
-    libkakasi2-dev
+    libkakasi2-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /root/osm2pgsql
 
@@ -121,17 +128,16 @@ RUN wget -O libutf8proc1.deb http://ftp.ch.debian.org/debian/pool/main/u/utf8pro
 RUN dpkg --install libutf8proc1.deb libutf8proc-dev.deb
 RUN rm libutf8proc1.deb libutf8proc-dev.deb
 
-RUN git clone https://github.com/giggls/mapnik-german-l10n.git mapnik-german-l10n
+RUN apt-get clean && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y pandoc \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-WORKDIR $CODE/mapnik-german-l10n/utf8translit
-RUN dpkg-buildpackage -uc -us -b
+RUN git clone https://github.com/giggls/mapnik-german-l10n.git mapnik-german-l10n
 
-WORKDIR $CODE/mapnik-german-l10n/kanjitranslit
-RUN dpkg-buildpackage -uc -us -b
+WORKDIR $CODE/mapnik-german-l10n
 
-WORKDIR $CODE/mapnik-german-l10n/
-RUN dpkg --install *utf8translit_*.deb
-RUN dpkg --install *kanjitranslit_*.deb
+RUN make && make install && make clean
 
 ENV HOME /home/py
 
@@ -151,13 +157,15 @@ RUN sed -i '1ilocal   all             all                                     tr
 
 RUN chmod a+rx $CODE
 
-WORKDIR $HOME
+WORKDIR /var/data/garmin/additional_data/
+# Fetch required additional data for Garmin as documented http://www.mkgmap.org.uk/download/mkgmap.html
+RUN wget -O /var/data/garmin/additional_data/bounds.zip http://osm2.pleiades.uni-wuppertal.de/bounds/latest/bounds.zip
+RUN wget -O /var/data/garmin/additional_data/sea.zip http://osm2.pleiades.uni-wuppertal.de/sea/latest/sea.zip
 
-ENV REQS_LAST_UPDATED 04-05-2016 8:50
-ADD ./requirements.txt $HOME/requirements.txt
+WORKDIR $HOME
 
 RUN pip3 install honcho
-
+ADD ./requirements.txt $HOME/requirements.txt
 RUN pip3 install -r requirements.txt
 
 # TODO: this is just a temporary solution, use pip for production as soon as geometalab.osmaxx is published there
@@ -167,6 +175,7 @@ ADD ./osmaxx_conversion_service $HOME/osmaxx_conversion_service
 # expose modules
 ENV PYTHONPATH=PYTHONPATH:$HOME
 ENV DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.worker
+ENV WORKER_QUEUES default high
 
 ENTRYPOINT ["/home/py/entrypoint/entrypoint.sh"]
 

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2016 HSR Hochschule für Technik, Rapperswil
+Copyright (c) 2015-2016 HSR Hochschule für Technik Rapperswil
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -43,6 +43,29 @@ compose-env/frontend.env: compose-env-dist/frontend.env
 	    < $< \
 	    > $@
 
+PIP_TOOLS_SOURCE_SPEC_FILES := requirements-all.in requirements-local.in requirements-mediator.in
+PIP_TOOLS_COMPILED_SPEC_FILES := $(PIP_TOOLS_SOURCE_SPEC_FILES:.in=.txt)
+
+.PHONY: pip-upgrade
+pip-upgrade: $(PIP_TOOLS_SOURCE_SPEC_FILES)
+	$(MAKE) --always-make $(PIP_TOOLS_COMPILED_SPEC_FILES)
+	@echo
+	@echo Updated compiled pip-tools spec files $<, but NOT INSTALLED, yet.
+	@echo Consider running
+	@echo "\t"make pip-sync-all
+	@echo or
+	@echo "\tpip-sync <compiled spec file> [<compiled spec file> ...]"
+	@echo e.g.
+	@echo "\t"pip-sync $(PIP_TOOLS_COMPILED_SPEC_FILES)
+	@echo now.
+
+.PHONY: pip-sync-all
+pip-sync-all: requirements-all.txt
+	pip-sync $?
+
+%.txt: %.in
+	pip-compile --output-file $@ $<
+
 compose-env/%.env: compose-env-dist/%.env
 	@mkdir -p $(@D)
 # We don't have to set DJANGO_SECRET_KEY here, as docker-compose-dev.yml sets it for local use.

build_and_push_images.py

@@ -10,7 +10,7 @@
 
 
 def docker_build(dockerfile, image_name, release, location='.'):
-    subprocess.check_call(['docker', 'build', '-f', dockerfile, '-t', '{}:{}'.format(image_name, release), location])
+    subprocess.check_call(['docker', 'build', '--pull', '-f', dockerfile, '-t', '{}:{}'.format(image_name, release), location])
 
 
 def docker_push(release, image_name, *args, **kwargs):

docker-compose.yml

@@ -8,6 +8,8 @@ services:
       - frontend
     networks:
       - osmaxx-internal
+    environment:
+      - VIRTUAL_HOST=localhost,127.0.0.1
   # if you change the names of the containers, please remember
   # that you need to adapt REAMDE.md
   ##### frontend START ########
@@ -106,6 +108,35 @@ services:
       - osm-world
       - osmboundaries
       - osmaxx-internal
+  worker-exclusive:
+    image: geometalab/osmaxx-worker:${DEPLOY_VERSION}
+    extends:
+      file: docker-compose-common.yml
+      service: conversionbase
+    command: [honcho, -f, ./osmaxx_conversion_service/Procfile.worker, start]
+    volumes:
+      - osm_data:/var/data/osm-planet
+      - worker-data:/data/media/job_result_files
+    links:
+      - conversionserviceredis:redis
+      - world-database
+      - osmboundaries-database
+    env_file:
+      - ./compose-env/worker.env
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - DJANGO_SETTINGS_MODULE=osmaxx_conversion_service.config.settings.worker
+      # only listens for jobs in this specific queue
+      - WORKER_QUEUES=high
+    depends_on:
+      - conversionserviceredis
+      - world-database
+      - osmboundaries-database
+    networks:
+      - osm-world
+      - osmboundaries
+      - osmaxx-internal
   conversionserviceredis:
     image: redis
     networks:

docker_entrypoint/nginx/default.conf.template

@@ -0,0 +1,67 @@
+# read more here http://lollyrock.com/articles/content-security-policy/
+
+# Copied and changed from https://gist.github.com/plentz/6737338
+
+# don't send the nginx version number in error pages and Server header
+server_tokens off;
+
+# config to don't allow the browser to render the page inside an frame or iframe
+# and avoid clickjacking http://en.wikipedia.org/wiki/Clickjacking
+# if you need to allow [i]frames, you can use SAMEORIGIN or even set an uri with ALLOW-FROM uri
+# https://developer.mozilla.org/en-US/docs/HTTP/X-Frame-Options
+add_header X-Frame-Options SAMEORIGIN;
+
+# when serving user-supplied content, include a X-Content-Type-Options: nosniff header along with the Content-Type: header,
+# to disable content-type sniffing on some browsers.
+# https://www.owasp.org/index.php/List_of_useful_HTTP_headers
+# currently suppoorted in IE > 8 http://blogs.msdn.com/b/ie/archive/2008/09/02/ie8-security-part-vi-beta-2-update.aspx
+# http://msdn.microsoft.com/en-us/library/ie/gg622941(v=vs.85).aspx
+# 'soon' on Firefox https://bugzilla.mozilla.org/show_bug.cgi?id=471020
+add_header X-Content-Type-Options nosniff;
+
+# This header enables the Cross-site scripting (XSS) filter built into most recent web browsers.
+# It's usually enabled by default anyway, so the role of this header is to re-enable the filter for
+# this particular website if it was disabled by the user.
+# https://www.owasp.org/index.php/List_of_useful_HTTP_headers
+add_header X-XSS-Protection "1; mode=block";
+
+# with Content Security Policy (CSP) enabled(and a browser that supports it(http://caniuse.com/#feat=contentsecuritypolicy),
+# you can tell the browser that it can only download content from the domains you explicitly allow
+# http://www.html5rocks.com/en/tutorials/security/content-security-policy/
+# https://www.owasp.org/index.php/Content_Security_Policy
+# more: http://www.html5rocks.com/en/tutorials/security/content-security-policy/#inline-code-considered-harmful
+add_header Content-Security-Policy "default-src 'self'; script-src 'self'; img-src 'self' http://*.osm.org https://*.osm.org data:; style-src 'self' 'unsafe-inline'; font-src 'self' data:; frame-src; object-src 'none'";
+
+proxy_http_version 1.1;
+proxy_buffering off;
+proxy_set_header Host $http_host;
+proxy_set_header Upgrade $http_upgrade;
+proxy_set_header X-Real-IP $remote_addr;
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+server {
+    server_name _; # This is just an invalid value which will never trigger on a real hostname.
+    listen 80;
+    access_log /var/log/nginx/access.log;
+    return 503;
+}
+
+upstream frontend {
+    server frontend:8000;
+}
+
+server {
+    listen 80;
+    server_name ${DOMAIN_NAMES};
+
+    location = /favicon.ico { access_log off; log_not_found off; }
+
+    location / {
+        proxy_pass http://frontend;
+    }
+
+    location /media {
+        autoindex off;
+        root /data/frontend/;
+    }
+}

docker_entrypoint/osmaxx/worker/01_activate_translit.sql

@@ -1 +1,2 @@
-CREATE FUNCTION transliterate(text) RETURNS text AS '$libdir/utf8translit', 'transliterate' LANGUAGE C STRICT;
+CREATE EXTENSION IF NOT EXISTS postgis;
+CREATE EXTENSION IF NOT EXISTS osml10n;