Skip to content
This repository has been archived by the owner on Apr 16, 2023. It is now read-only.

Commit

Permalink
Merge pull request #868 from geometalab/feature/#845-conversion_status
Browse files Browse the repository at this point in the history
Feature/#845 conversion status
  • Loading branch information
hixi authored May 19, 2017
2 parents 53b452c + 99f6b11 commit e27f1e8
Show file tree
Hide file tree
Showing 19 changed files with 96 additions and 83 deletions.
4 changes: 2 additions & 2 deletions osmaxx/conversion/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from .constants import output_format
from .constants import output_format, status

default_app_config = 'osmaxx.conversion.apps.ConversionConfig'

__all__ = ['default_app_config', 'output_format']
__all__ = ['default_app_config', 'output_format', 'status']
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
STARTED = JobStatus.STARTED
DEFERRED = JobStatus.DEFERRED

STATUS_CHOICES = (
CHOICES = (
(RECEIVED, _('received')),
# these are identical to the job-statuses of rq
(JobStatus.QUEUED, _('queued')),
Expand Down
13 changes: 6 additions & 7 deletions osmaxx/conversion/management/commands/result_harvester.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,8 @@
from django.core.management.base import BaseCommand
from pbf_file_size_estimation import estimate_size

from osmaxx.conversion import models as conversion_models
from osmaxx.conversion import models as conversion_models, status
from osmaxx.conversion._settings import CONVERSION_SETTINGS
from osmaxx.conversion.constants.statuses import FINAL_STATUSES, FINISHED, FAILED

logging.basicConfig()
logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -42,7 +41,7 @@ def _handle_failed_jobs(self):
self._notify(conversion_job)

def _handle_running_jobs(self):
active_jobs = conversion_models.Job.objects.exclude(status__in=FINAL_STATUSES)\
active_jobs = conversion_models.Job.objects.exclude(status__in=status.FINAL_STATUSES)\
.values_list('rq_job_id', flat=True)
for job_id in active_jobs:
self._update_job(rq_job_id=job_id)
Expand All @@ -67,19 +66,19 @@ def _update_job(self, rq_job_id):

logger.info('updating job %d', rq_job_id)
conversion_job.status = job.status
if job.status == FINISHED:
if job.status == status.FINISHED:
add_file_to_job(conversion_job=conversion_job, result_zip_file=job.kwargs['output_zip_file_path'])
add_meta_data_to_job(conversion_job=conversion_job, rq_job=job)
conversion_job.save()
self._notify(conversion_job)

def _set_failed_unless_final(self, conversion_job, rq_job_id):
conversion_job.refresh_from_db()
if conversion_job.status not in FINAL_STATUSES:
if conversion_job.status not in status.FINAL_STATUSES:
logger.error("job {} of conversion job {} not found in queue but status is {} on database.".format(
rq_job_id, conversion_job.id, conversion_job.status
))
conversion_job.status = FAILED
conversion_job.status = status.FAILED
conversion_job.save()

def _notify(self, conversion_job):
Expand Down Expand Up @@ -136,5 +135,5 @@ def cleanup_old_jobs():
queues.append(django_rq.get_failed_queue())
for queue in queues:
for job in queue.get_jobs():
if job.status in FINAL_STATUSES:
if job.status in status.FINAL_STATUSES:
job.delete()
5 changes: 2 additions & 3 deletions osmaxx/conversion/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,11 @@
from django.utils.translation import gettext_lazy as _
from rest_framework.reverse import reverse

from osmaxx.conversion import output_format
from osmaxx.conversion import output_format, status
from osmaxx.clipping_area.models import ClippingArea
from osmaxx.conversion.converters.converter import convert
from osmaxx.conversion.converters.converter_gis.detail_levels import DETAIL_LEVEL_CHOICES, DETAIL_LEVEL_ALL
from osmaxx.conversion.constants.coordinate_reference_systems import CRS_CHOICES
from osmaxx.conversion.constants.statuses import STATUS_CHOICES, RECEIVED


def job_directory_path(instance, filename):
Expand Down Expand Up @@ -40,7 +39,7 @@ class Job(models.Model):
callback_url = models.URLField(_('callback url'), max_length=250)
parametrization = models.ForeignKey(verbose_name=_('parametrization'), to=Parametrization, on_delete=models.CASCADE)
rq_job_id = models.CharField(_('rq job id'), max_length=250, null=True)
status = models.CharField(_('job status'), choices=STATUS_CHOICES, default=RECEIVED, max_length=20)
status = models.CharField(_('job status'), choices=status.CHOICES, default=status.RECEIVED, max_length=20)
resulting_file = models.FileField(_('resulting file'), upload_to=job_directory_path, null=True, max_length=250)
estimated_pbf_size = models.FloatField(_('estimated pbf size in bytes'), null=True)
unzipped_result_size = models.FloatField(
Expand Down
20 changes: 20 additions & 0 deletions osmaxx/excerptexport/migrations/0058_auto_20170511_1639.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.7 on 2017-05-11 14:39
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('excerptexport', '0057_auto_20170511_1406'),
]

operations = [
migrations.AlterField(
model_name='export',
name='status',
field=models.CharField(choices=[('received', 'received'), ('queued', 'queued'), ('finished', 'finished'), ('failed', 'failed'), ('started', 'started'), ('deferred', 'deferred')], default=None, max_length=20, null=True, verbose_name='job status'),
),
]
32 changes: 14 additions & 18 deletions osmaxx/excerptexport/models/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from django.utils.translation import ugettext_lazy as _
from rest_framework.reverse import reverse

from osmaxx.conversion import output_format
from osmaxx.conversion import output_format, status
from osmaxx.excerptexport._settings import RESULT_FILE_AVAILABILITY_DURATION, EXTRACTION_PROCESSING_TIMEOUT_TIMEDELTA

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -41,16 +41,12 @@ class Export(TimeStampModelMixin, models.Model):
- the transformation of the data from the data sources' schemata (e.g. ``osm2pgsql`` schema) to the OSMaxx schema
- the actual export to one specific GIS or navigation file format with one specific set of parameters
"""
from osmaxx.conversion.constants.statuses import RECEIVED, QUEUED, FINISHED, FAILED, STARTED, DEFERRED, FINAL_STATUSES, STATUS_CHOICES # noqa
INITIAL = 'initial'
INITIAL_CHOICE = (INITIAL, _('initial'))
STATUS_CHOICES = (INITIAL_CHOICE,) + STATUS_CHOICES

extraction_order = models.ForeignKey('excerptexport.ExtractionOrder', related_name='exports',
verbose_name=_('extraction order'), on_delete=models.CASCADE)
file_format = models.CharField(choices=output_format.CHOICES, verbose_name=_('file format / data format'), max_length=10)
conversion_service_job_id = models.IntegerField(verbose_name=_('conversion service job ID'), null=True)
status = models.CharField(_('job status'), choices=STATUS_CHOICES, default=INITIAL, max_length=20)
status = models.CharField(_('job status'), choices=status.CHOICES, default=None, max_length=20, null=True)
finished_at = models.DateTimeField(_('finished at'), default=None, blank=True, editable=False, null=True)

def delete(self, *args, **kwargs):
Expand Down Expand Up @@ -81,9 +77,9 @@ def status_update_url(self):
return reverse('job_progress:tracker', kwargs=dict(export_id=self.id))

def set_and_handle_new_status(self, new_status, *, incoming_request):
assert new_status in dict(self.STATUS_CHOICES)
assert new_status in dict(status.CHOICES) or new_status is None
if self.status == new_status and self.update_is_overdue:
new_status = self.FAILED
new_status = status.FAILED

if self.status != new_status:
self.status = new_status
Expand All @@ -100,9 +96,9 @@ def _handle_changed_status(self, *, incoming_request):
from osmaxx.utils.shortcuts import Emissary
emissary = Emissary(recipient=self.extraction_order.orderer)
status_changed_message = self._get_export_status_changed_message()
if self.status == self.FAILED:
if self.status == status.FAILED:
emissary.error(status_changed_message)
elif self.status == self.FINISHED:
elif self.status == status.FINISHED:
from osmaxx.api_client.conversion_api_client import ResultFileNotAvailableError
try:
self._fetch_result_file()
Expand Down Expand Up @@ -160,7 +156,7 @@ def result_file_available_until(self):

@property
def is_status_final(self):
return self.status in self.FINAL_STATUSES
return self.status in status.FINAL_STATUSES

@property
def can_be_deleted(self):
Expand All @@ -170,7 +166,7 @@ def can_be_deleted(self):
def is_running(self):
if self.update_is_overdue:
now = timezone.now()
self.status = self.FAILED
self.status = status.FAILED
self.finished_at = now
self.save()
return not self.is_status_final
Expand All @@ -185,11 +181,11 @@ def css_status_class(self):
default_class = 'default'

status_map = {
self.RECEIVED: 'info',
self.QUEUED: 'info',
self.FINISHED: 'success',
self.FAILED: 'danger',
self.STARTED: 'info',
self.DEFERRED: 'default',
status.RECEIVED: 'info',
status.QUEUED: 'info',
status.FINISHED: 'success',
status.FAILED: 'danger',
status.STARTED: 'info',
status.DEFERRED: 'default',
}
return status_map.get(self.status, default_class)
6 changes: 3 additions & 3 deletions osmaxx/excerptexport/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from django.views.generic.list import ListView

from osmaxx.contrib.auth.frontend_permissions import EmailRequiredMixin
from osmaxx.conversion.constants import statuses
from osmaxx.conversion import status
from osmaxx.excerptexport.forms import ExcerptForm, ExistingForm
from osmaxx.excerptexport.models import Excerpt
from osmaxx.excerptexport.models import ExtractionOrder
Expand Down Expand Up @@ -70,7 +70,7 @@ def get_object(self, queryset=None):


class ExportsListMixin:
_filterable_statuses = frozenset({statuses.FINISHED, statuses.FAILED})
_filterable_statuses = frozenset({status.FINISHED, status.FAILED})

@property
def excerpt_ids(self):
Expand All @@ -85,7 +85,7 @@ def excerpt_ids(self):

@property
def status_choices(self):
return [choice for choice in Export.STATUS_CHOICES if choice[0] in self._filterable_statuses]
return [choice for choice in status.CHOICES if choice[0] in self._filterable_statuses]

def get_user_exports(self):
return self._filter_exports(
Expand Down
10 changes: 5 additions & 5 deletions osmaxx/job_progress/middleware.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from requests import HTTPError

from osmaxx.api_client import ConversionApiClient
from osmaxx.conversion.constants.statuses import FINAL_STATUSES, FAILED
from osmaxx.conversion import status
from osmaxx.excerptexport.models import Export
from osmaxx.utils.shortcuts import get_cached_or_set

Expand Down Expand Up @@ -34,10 +34,10 @@ def process_request(self, request):

def handle_unsent_exports(user):
for export in Export.objects.\
exclude(status__in=FINAL_STATUSES).\
exclude(status__in=status.FINAL_STATUSES).\
filter(extraction_order__orderer=user, conversion_service_job_id__isnull=True):
if export.update_is_overdue:
export.status = export.FAILED
export.status = status.FAILED
export.save()


Expand All @@ -49,15 +49,15 @@ def update_exports_of_request_user(request):
handle_unsent_exports(user=current_user)

pending_exports = Export.objects.\
exclude(status__in=FINAL_STATUSES).\
exclude(status__in=status.FINAL_STATUSES).\
filter(extraction_order__orderer=current_user, conversion_service_job_id__isnull=False)
for export in pending_exports:
try:
update_export_if_stale(export, request=request)
except HTTPError as e:
if e.response.status_code == requests.codes['not_found']:
logger.exception("Export #%s doesn't exist on the conversion service.", export.id)
export.status = FAILED
export.status = status.FAILED
export.save()
else:
logger.exception("Failed to update status of pending export #%s.", export.id)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{% autoescape off %}
Export #{{ export.id }} "{{ export.extraction_order.excerpt_name }}" to {{ export.get_file_format_display }} has {% if not export.is_status_final %}been {% endif %}{{ export.status }}.{% if export.status == export.STARTED %} Exporting will take around 30 minutes.{% endif %}
Export #{{ export.id }} "{{ export.extraction_order.excerpt_name }}" to {{ export.get_file_format_display }} has {% if not export.is_status_final %}been {% endif %}{{ export.status }}.{% if export.status == 'started' %} Exporting will take around 30 minutes.{% endif %}
{% endautoescape %}
11 changes: 5 additions & 6 deletions tests/conversion/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@
import pytest
from django.conf import settings

from osmaxx.conversion import output_format
from osmaxx.conversion import output_format, status
from osmaxx.conversion.converters.converter_gis import detail_levels
from osmaxx.conversion.constants import coordinate_reference_systems as crs
from osmaxx.conversion.constants.statuses import STARTED, FAILED, FINISHED

format_list = output_format.DEFINITIONS.keys()

Expand Down Expand Up @@ -116,20 +115,20 @@ def conversion_job(conversion_parametrization, server_url):
@pytest.fixture
def started_conversion_job(conversion_parametrization, server_url, fake_rq_id):
from osmaxx.conversion.models import Job
return Job.objects.create(own_base_url=server_url, parametrization=conversion_parametrization, rq_job_id=fake_rq_id, status=STARTED)
return Job.objects.create(own_base_url=server_url, parametrization=conversion_parametrization, rq_job_id=fake_rq_id, status=status.STARTED)


@pytest.fixture
def failed_conversion_job(conversion_parametrization, server_url, fake_rq_id):
from osmaxx.conversion.models import Job
return Job.objects.create(own_base_url=server_url, parametrization=conversion_parametrization, rq_job_id=fake_rq_id, status=FAILED)
return Job.objects.create(own_base_url=server_url, parametrization=conversion_parametrization, rq_job_id=fake_rq_id, status=status.FAILED)


@pytest.fixture
def finished_conversion_job(request, conversion_parametrization, server_url, fake_rq_id, empty_zip):
from osmaxx.conversion.models import Job
conversion_job = Job.objects.create(own_base_url=server_url, parametrization=conversion_parametrization, rq_job_id=fake_rq_id, status=FAILED)
conversion_job.status = FINISHED
conversion_job = Job.objects.create(own_base_url=server_url, parametrization=conversion_parametrization, rq_job_id=fake_rq_id, status=status.FAILED)
conversion_job.status = status.FINISHED
from osmaxx.conversion.management.commands.result_harvester import add_file_to_job
empty_zip_path = add_file_to_job(conversion_job=conversion_job, result_zip_file=empty_zip.name)

Expand Down
4 changes: 2 additions & 2 deletions tests/conversion/management/commands/result_harvester_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import pytest

from osmaxx.conversion.constants.statuses import STARTED
from osmaxx.conversion import status


@pytest.fixture
Expand All @@ -12,7 +12,7 @@ def queue(fake_rq_id):
Job = namedtuple('Job', ['status'])

def fetch_job(job_id):
job = Job(status=STARTED)
job = Job(status=status.STARTED)
return job
queue = Queue(job_ids=[str(fake_rq_id)], fetch_job=fetch_job)
return queue
Expand Down
10 changes: 5 additions & 5 deletions tests/conversion/model_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,19 @@

import pytest

from osmaxx.conversion.constants.statuses import STARTED, FAILED, FINISHED
from osmaxx.conversion import status


@pytest.mark.django_db()
def test_job_statuses_for_jobs(started_conversion_job, failed_conversion_job, finished_conversion_job):
assert started_conversion_job.status == STARTED
assert failed_conversion_job.status == FAILED
assert finished_conversion_job.status == FINISHED
assert started_conversion_job.status == status.STARTED
assert failed_conversion_job.status == status.FAILED
assert finished_conversion_job.status == status.FINISHED


@pytest.mark.django_db()
def test_job_removes_file_when_deleted(finished_conversion_job):
assert finished_conversion_job.status == FINISHED
assert finished_conversion_job.status == status.FINISHED
file_path = finished_conversion_job.resulting_file.path
assert file_path is not None
assert os.path.exists(file_path)
Expand Down
4 changes: 2 additions & 2 deletions tests/conversion/view_test.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import pytest
from rest_framework.reverse import reverse

from osmaxx.conversion.constants.statuses import RECEIVED
from osmaxx.conversion import status

authenticated_access_urls = [
reverse('clipping_area-list'),
Expand Down Expand Up @@ -85,7 +85,7 @@ def test_conversion_job_detail_access_success(authenticated_api_client, conversi
assert data['id'] == conversion_job.id
assert data['callback_url'] == conversion_job.callback_url
assert data['parametrization'] == conversion_parametrization.id
assert data['status'] == RECEIVED
assert data['status'] == status.RECEIVED
assert data['resulting_file_path'] is None


Expand Down
Loading

0 comments on commit e27f1e8

Please sign in to comment.